2024-04-02T14:47:22.943 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-04-02T14:47:22.944 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_gabrioux@teuthology --archive /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636039 --name gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi --verbose --description orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} -- /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636039/orig.config.yaml 2024-04-02T14:47:22.987 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-02T14:47:23.062 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636039 branch: wip-guits-testing-2024-04-02-0703 description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} email: gabrioux@ibm.com first_in_suite: false job_id: '7636039' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-testing-2024-04-02-0703 ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 workunit: branch: wip-guits-testing-2024-04-02-0703 sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 5637 sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-testing-2024-04-02-0703 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 targets: smithi042.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKpnlmMUlvy2mfaktOdc7N9aQbBz7qTP7ch5a3MsEvt3PxsGNa5DgrrQtZ7KKeWsXqT0iw243939Ae8VWa2J8CA= smithi169.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEBuLnltjtfvWtb3uWYalmbxm0omnfwG45avdad+XyG42yEv9QBPNeW2LNIp8DKlYhldm4psBIr49Y0sdKzBoNk= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-04-02_14:30:22 tube: smithi user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-02T14:47:23.063 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa; will attempt to use it 2024-04-02T14:47:23.063 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa/tasks 2024-04-02T14:47:23.064 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-02T14:47:23.065 INFO:teuthology.task.internal:Checking packages... 2024-04-02T14:47:23.088 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '3f3f3ed992429b85be729743ad3b39d52dd90744' 2024-04-02T14:47:23.088 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-02T14:47:23.088 INFO:teuthology.packaging:ref: None 2024-04-02T14:47:23.088 INFO:teuthology.packaging:tag: None 2024-04-02T14:47:23.088 INFO:teuthology.packaging:branch: wip-guits-testing-2024-04-02-0703 2024-04-02T14:47:23.088 INFO:teuthology.packaging:sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 2024-04-02T14:47:23.089 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-testing-2024-04-02-0703 2024-04-02T14:47:23.226 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2592.g3f3f3ed9 2024-04-02T14:47:23.227 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-02T14:47:23.234 INFO:teuthology.task.internal:no buildpackages task found 2024-04-02T14:47:23.234 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-02T14:47:23.249 INFO:teuthology.task.internal:Saving configuration 2024-04-02T14:47:23.267 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-02T14:47:23.273 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-02T14:47:23.294 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi042.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636039', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 14:42:05.927714', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKpnlmMUlvy2mfaktOdc7N9aQbBz7qTP7ch5a3MsEvt3PxsGNa5DgrrQtZ7KKeWsXqT0iw243939Ae8VWa2J8CA='} 2024-04-02T14:47:23.313 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi169.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636039', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 14:42:05.929639', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEBuLnltjtfvWtb3uWYalmbxm0omnfwG45avdad+XyG42yEv9QBPNeW2LNIp8DKlYhldm4psBIr49Y0sdKzBoNk='} 2024-04-02T14:47:23.313 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-02T14:47:23.319 INFO:teuthology.task.internal:roles: ubuntu@smithi042.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-02T14:47:23.319 INFO:teuthology.task.internal:roles: ubuntu@smithi169.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-02T14:47:23.320 INFO:teuthology.run_tasks:Running task console_log... 2024-04-02T14:47:23.387 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f3a4031d0d0>, signals=[15]) 2024-04-02T14:47:23.387 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-02T14:47:23.394 INFO:teuthology.task.internal:Opening connections... 2024-04-02T14:47:23.394 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi042.front.sepia.ceph.com 2024-04-02T14:47:23.396 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi042.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:47:23.514 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi169.front.sepia.ceph.com 2024-04-02T14:47:23.515 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:47:23.602 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-02T14:47:23.609 DEBUG:teuthology.orchestra.run.smithi042:> uname -m 2024-04-02T14:47:23.627 INFO:teuthology.orchestra.run.smithi042.stdout:x86_64 2024-04-02T14:47:23.627 DEBUG:teuthology.orchestra.run.smithi042:> cat /etc/os-release 2024-04-02T14:47:23.684 INFO:teuthology.orchestra.run.smithi042.stdout:NAME="CentOS Stream" 2024-04-02T14:47:23.684 INFO:teuthology.orchestra.run.smithi042.stdout:VERSION="9" 2024-04-02T14:47:23.684 INFO:teuthology.orchestra.run.smithi042.stdout:ID="centos" 2024-04-02T14:47:23.684 INFO:teuthology.orchestra.run.smithi042.stdout:ID_LIKE="rhel fedora" 2024-04-02T14:47:23.684 INFO:teuthology.orchestra.run.smithi042.stdout:VERSION_ID="9" 2024-04-02T14:47:23.684 INFO:teuthology.orchestra.run.smithi042.stdout:PLATFORM_ID="platform:el9" 2024-04-02T14:47:23.684 INFO:teuthology.orchestra.run.smithi042.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-02T14:47:23.684 INFO:teuthology.orchestra.run.smithi042.stdout:ANSI_COLOR="0;31" 2024-04-02T14:47:23.684 INFO:teuthology.orchestra.run.smithi042.stdout:LOGO="fedora-logo-icon" 2024-04-02T14:47:23.685 INFO:teuthology.orchestra.run.smithi042.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-02T14:47:23.685 INFO:teuthology.orchestra.run.smithi042.stdout:HOME_URL="https://centos.org/" 2024-04-02T14:47:23.685 INFO:teuthology.orchestra.run.smithi042.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-02T14:47:23.685 INFO:teuthology.orchestra.run.smithi042.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-02T14:47:23.685 INFO:teuthology.orchestra.run.smithi042.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-02T14:47:23.685 INFO:teuthology.lock.ops:Updating smithi042.front.sepia.ceph.com on lock server 2024-04-02T14:47:23.702 DEBUG:teuthology.orchestra.run.smithi169:> uname -m 2024-04-02T14:47:23.719 INFO:teuthology.orchestra.run.smithi169.stdout:x86_64 2024-04-02T14:47:23.719 DEBUG:teuthology.orchestra.run.smithi169:> cat /etc/os-release 2024-04-02T14:47:23.778 INFO:teuthology.orchestra.run.smithi169.stdout:NAME="CentOS Stream" 2024-04-02T14:47:23.778 INFO:teuthology.orchestra.run.smithi169.stdout:VERSION="9" 2024-04-02T14:47:23.778 INFO:teuthology.orchestra.run.smithi169.stdout:ID="centos" 2024-04-02T14:47:23.778 INFO:teuthology.orchestra.run.smithi169.stdout:ID_LIKE="rhel fedora" 2024-04-02T14:47:23.778 INFO:teuthology.orchestra.run.smithi169.stdout:VERSION_ID="9" 2024-04-02T14:47:23.778 INFO:teuthology.orchestra.run.smithi169.stdout:PLATFORM_ID="platform:el9" 2024-04-02T14:47:23.778 INFO:teuthology.orchestra.run.smithi169.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-02T14:47:23.779 INFO:teuthology.orchestra.run.smithi169.stdout:ANSI_COLOR="0;31" 2024-04-02T14:47:23.779 INFO:teuthology.orchestra.run.smithi169.stdout:LOGO="fedora-logo-icon" 2024-04-02T14:47:23.779 INFO:teuthology.orchestra.run.smithi169.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-02T14:47:23.779 INFO:teuthology.orchestra.run.smithi169.stdout:HOME_URL="https://centos.org/" 2024-04-02T14:47:23.779 INFO:teuthology.orchestra.run.smithi169.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-02T14:47:23.779 INFO:teuthology.orchestra.run.smithi169.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-02T14:47:23.779 INFO:teuthology.orchestra.run.smithi169.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-02T14:47:23.779 INFO:teuthology.lock.ops:Updating smithi169.front.sepia.ceph.com on lock server 2024-04-02T14:47:23.801 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-02T14:47:23.811 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-02T14:47:23.817 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-02T14:47:23.817 DEBUG:teuthology.orchestra.run.smithi042:> test '!' -e /home/ubuntu/cephtest 2024-04-02T14:47:23.819 DEBUG:teuthology.orchestra.run.smithi169:> test '!' -e /home/ubuntu/cephtest 2024-04-02T14:47:23.836 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-02T14:47:23.841 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-02T14:47:23.841 DEBUG:teuthology.orchestra.run.smithi042:> test -z $(ls -A /var/lib/ceph) 2024-04-02T14:47:23.875 DEBUG:teuthology.orchestra.run.smithi169:> test -z $(ls -A /var/lib/ceph) 2024-04-02T14:47:24.001 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-02T14:47:24.047 INFO:teuthology.run_tasks:Running task kernel... 2024-04-02T14:47:24.061 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-04-02T14:47:24.061 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-04-02T14:47:24.061 DEBUG:teuthology.orchestra.run.smithi042:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-02T14:47:24.062 DEBUG:teuthology.orchestra.run.smithi169:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-02T14:47:24.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T14:47:24.078 DEBUG:teuthology.orchestra.run.smithi042:> uname -r 2024-04-02T14:47:24.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T14:47:24.080 DEBUG:teuthology.orchestra.run.smithi169:> uname -r 2024-04-02T14:47:24.136 INFO:teuthology.orchestra.run.smithi042.stdout:5.14.0-361.el9.x86_64 2024-04-02T14:47:24.136 INFO:teuthology.task.kernel:Running kernel on smithi042: 5.14.0-361.el9.x86_64 2024-04-02T14:47:24.136 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum install -y kernel 2024-04-02T14:47:24.137 INFO:teuthology.orchestra.run.smithi169.stdout:5.14.0-361.el9.x86_64 2024-04-02T14:47:24.137 INFO:teuthology.task.kernel:Running kernel on smithi169: 5.14.0-361.el9.x86_64 2024-04-02T14:47:24.137 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum install -y kernel 2024-04-02T14:47:26.201 INFO:teuthology.orchestra.run.smithi169.stdout:CentOS-9 - AppStream 15 MB/s | 6.7 MB 00:00 2024-04-02T14:47:26.284 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS-9 - AppStream 13 MB/s | 6.7 MB 00:00 2024-04-02T14:47:27.742 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS-9 - BaseOS 6.9 MB/s | 2.0 MB 00:00 2024-04-02T14:47:27.818 INFO:teuthology.orchestra.run.smithi169.stdout:CentOS-9 - BaseOS 3.9 MB/s | 2.0 MB 00:00 2024-04-02T14:47:28.584 INFO:teuthology.orchestra.run.smithi169.stdout:CentOS Stream 9 - BaseOS 18 MB/s | 8.0 MB 00:00 2024-04-02T14:47:28.949 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - BaseOS 9.3 MB/s | 8.0 MB 00:00 2024-04-02T14:47:30.444 INFO:teuthology.orchestra.run.smithi169.stdout:CentOS Stream 9 - AppStream 24 MB/s | 19 MB 00:00 2024-04-02T14:47:31.867 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - AppStream 11 MB/s | 19 MB 00:01 2024-04-02T14:47:34.520 INFO:teuthology.orchestra.run.smithi169.stdout:CentOS Stream 9 - CRB 12 MB/s | 6.1 MB 00:00 2024-04-02T14:47:36.137 INFO:teuthology.orchestra.run.smithi169.stdout:CentOS Stream 9 - Extras packages 33 kB/s | 16 kB 00:00 2024-04-02T14:47:36.168 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.1 MB 00:00 2024-04-02T14:47:37.480 INFO:teuthology.orchestra.run.smithi169.stdout:Extra Packages for Enterprise Linux 17 MB/s | 21 MB 00:01 2024-04-02T14:47:37.610 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - Extras packages 63 kB/s | 16 kB 00:00 2024-04-02T14:47:38.512 INFO:teuthology.orchestra.run.smithi042.stdout:Extra Packages for Enterprise Linux 27 MB/s | 21 MB 00:00 2024-04-02T14:47:42.555 INFO:teuthology.orchestra.run.smithi169.stdout:lab-extras 15 kB/s | 1.7 kB 00:00 2024-04-02T14:47:43.729 INFO:teuthology.orchestra.run.smithi042.stdout:lab-extras 37 kB/s | 1.7 kB 00:00 2024-04-02T14:47:44.795 INFO:teuthology.orchestra.run.smithi169.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-02T14:47:44.795 INFO:teuthology.orchestra.run.smithi169.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-02T14:47:44.897 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2024-04-02T14:47:44.908 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2024-04-02T14:47:44.908 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2024-04-02T14:47:44.908 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2024-04-02T14:47:44.909 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2024-04-02T14:47:44.909 INFO:teuthology.orchestra.run.smithi169.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-04-02T14:47:44.909 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2024-04-02T14:47:44.909 INFO:teuthology.orchestra.run.smithi169.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-04-02T14:47:44.909 INFO:teuthology.orchestra.run.smithi169.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-04-02T14:47:44.909 INFO:teuthology.orchestra.run.smithi169.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-04-02T14:47:44.910 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-04-02T14:47:44.910 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2024-04-02T14:47:44.910 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2024-04-02T14:47:44.910 INFO:teuthology.orchestra.run.smithi169.stdout:Install 4 Packages 2024-04-02T14:47:44.910 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-04-02T14:47:44.910 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 76 M 2024-04-02T14:47:44.911 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 123 M 2024-04-02T14:47:44.911 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2024-04-02T14:47:45.064 INFO:teuthology.orchestra.run.smithi169.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 1.2 MB/s | 192 kB 00:00 2024-04-02T14:47:45.589 INFO:teuthology.orchestra.run.smithi169.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 22 MB/s | 15 MB 00:00 2024-04-02T14:47:46.052 INFO:teuthology.orchestra.run.smithi042.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-02T14:47:46.053 INFO:teuthology.orchestra.run.smithi042.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-02T14:47:46.152 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-04-02T14:47:46.161 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-04-02T14:47:46.162 INFO:teuthology.orchestra.run.smithi042.stdout: Package Arch Version Repository Size 2024-04-02T14:47:46.162 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-04-02T14:47:46.162 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-04-02T14:47:46.162 INFO:teuthology.orchestra.run.smithi042.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-04-02T14:47:46.162 INFO:teuthology.orchestra.run.smithi042.stdout:Installing dependencies: 2024-04-02T14:47:46.162 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-04-02T14:47:46.162 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-04-02T14:47:46.162 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-04-02T14:47:46.162 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T14:47:46.162 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-04-02T14:47:46.163 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-04-02T14:47:46.163 INFO:teuthology.orchestra.run.smithi042.stdout:Install 4 Packages 2024-04-02T14:47:46.163 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T14:47:46.163 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 76 M 2024-04-02T14:47:46.164 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 123 M 2024-04-02T14:47:46.164 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-04-02T14:47:46.322 INFO:teuthology.orchestra.run.smithi042.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 1.2 MB/s | 192 kB 00:00 2024-04-02T14:47:46.356 INFO:teuthology.orchestra.run.smithi169.stdout:(3/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 23 MB/s | 34 MB 00:01 2024-04-02T14:47:46.606 INFO:teuthology.orchestra.run.smithi169.stdout:(4/4): kernel-modules-core-5.14.0-434.el9.x86_6 18 MB/s | 28 MB 00:01 2024-04-02T14:47:46.610 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2024-04-02T14:47:46.610 INFO:teuthology.orchestra.run.smithi169.stdout:Total 45 MB/s | 76 MB 00:01 2024-04-02T14:47:46.610 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2024-04-02T14:47:46.914 INFO:teuthology.orchestra.run.smithi042.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 20 MB/s | 15 MB 00:00 2024-04-02T14:47:46.977 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2024-04-02T14:47:46.977 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2024-04-02T14:47:47.514 INFO:teuthology.orchestra.run.smithi042.stdout:(3/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 25 MB/s | 34 MB 00:01 2024-04-02T14:47:47.807 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2024-04-02T14:47:47.807 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2024-04-02T14:47:47.872 INFO:teuthology.orchestra.run.smithi042.stdout:(4/4): kernel-modules-core-5.14.0-434.el9.x86_6 18 MB/s | 28 MB 00:01 2024-04-02T14:47:47.876 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-04-02T14:47:47.876 INFO:teuthology.orchestra.run.smithi042.stdout:Total 45 MB/s | 76 MB 00:01 2024-04-02T14:47:47.876 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-04-02T14:47:48.319 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-04-02T14:47:48.319 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-04-02T14:47:49.206 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-04-02T14:47:49.206 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-04-02T14:47:50.098 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2024-04-02T14:47:50.696 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-04-02T14:47:50.782 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T14:47:51.629 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-04-02T14:47:52.263 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-04-02T14:47:52.349 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T14:47:53.569 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T14:47:53.711 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T14:47:55.249 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T14:47:55.379 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T14:47:58.439 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T14:47:58.476 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:48:00.486 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T14:48:00.520 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:48:03.236 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:48:05.758 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:48:32.441 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:48:32.441 INFO:teuthology.orchestra.run.smithi169.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-02T14:48:32.441 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-04-02T14:48:32.483 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:48:32.836 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:48:32.836 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-04-02T14:48:32.836 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T14:48:32.837 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T14:48:33.235 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:48:33.235 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-04-02T14:48:33.235 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2024-04-02T14:48:33.235 INFO:teuthology.orchestra.run.smithi169.stdout: kernel-5.14.0-434.el9.x86_64 2024-04-02T14:48:33.236 INFO:teuthology.orchestra.run.smithi169.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-04-02T14:48:33.236 INFO:teuthology.orchestra.run.smithi169.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-04-02T14:48:33.236 INFO:teuthology.orchestra.run.smithi169.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-04-02T14:48:33.236 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-04-02T14:48:33.236 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2024-04-02T14:48:33.567 DEBUG:teuthology.orchestra.run.smithi169:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T14:48:33.671 INFO:teuthology.orchestra.run.smithi169.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T14:48:33.672 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T14:48:33.672 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-04-02T14:48:33.672 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-02T14:48:33.672 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-02T14:48:33.672 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-02T14:48:33.672 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi169.front.sepia.ceph.com, path=None, version=distro) 2024-04-02T14:48:33.673 DEBUG:teuthology.orchestra.run.smithi169:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T14:48:33.753 INFO:teuthology.orchestra.run.smithi169.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T14:48:33.753 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T14:48:33.753 DEBUG:teuthology.orchestra.run.smithi169:> sudo rpm -qi grub2-tools 2024-04-02T14:48:33.794 INFO:teuthology.orchestra.run.smithi169.stdout:Name : grub2-tools 2024-04-02T14:48:33.794 INFO:teuthology.orchestra.run.smithi169.stdout:Epoch : 1 2024-04-02T14:48:33.794 INFO:teuthology.orchestra.run.smithi169.stdout:Version : 2.06 2024-04-02T14:48:33.794 INFO:teuthology.orchestra.run.smithi169.stdout:Release : 68.el9 2024-04-02T14:48:33.794 INFO:teuthology.orchestra.run.smithi169.stdout:Architecture: x86_64 2024-04-02T14:48:33.794 INFO:teuthology.orchestra.run.smithi169.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-02T14:48:33.794 INFO:teuthology.orchestra.run.smithi169.stdout:Group : Unspecified 2024-04-02T14:48:33.794 INFO:teuthology.orchestra.run.smithi169.stdout:Size : 8264254 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout:License : GPLv3+ 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout:Packager : builder@centos.org 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout:Vendor : CentOS 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout:Summary : Support tools for GRUB. 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout:Description : 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout:hardware devices. 2024-04-02T14:48:33.795 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-04-02T14:48:33.796 INFO:teuthology.orchestra.run.smithi169.stdout:This subpackage provides tools for support of all platforms. 2024-04-02T14:48:33.798 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-02T14:48:33.798 INFO:teuthology.task.kernel:Updating grub on smithi169 to boot 5.14.0-434.el9.x86_64 2024-04-02T14:48:33.799 DEBUG:teuthology.orchestra.run.smithi169:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-02T14:48:34.464 INFO:teuthology.orchestra.run.smithi169.stderr:Generating grub configuration file ... 2024-04-02T14:48:35.864 INFO:teuthology.orchestra.run.smithi169.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-02T14:48:35.900 INFO:teuthology.orchestra.run.smithi169.stderr:done 2024-04-02T14:48:35.902 DEBUG:teuthology.orchestra.run.smithi169:> mktemp 2024-04-02T14:48:35.918 INFO:teuthology.orchestra.run.smithi169.stdout:/tmp/tmp.JaF7BGca6v 2024-04-02T14:48:35.918 DEBUG:teuthology.orchestra.run.smithi169:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.JaF7BGca6v 2024-04-02T14:48:35.987 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod 0666 /tmp/tmp.JaF7BGca6v 2024-04-02T14:48:36.121 DEBUG:teuthology.orchestra.remote:smithi169:/tmp/tmp.JaF7BGca6v is 6KB 2024-04-02T14:48:36.141 DEBUG:teuthology.orchestra.run.smithi169:> rm -fr /tmp/tmp.JaF7BGca6v 2024-04-02T14:48:36.155 DEBUG:teuthology.orchestra.run.smithi169:> sudo /bin/ls /boot/loader/entries || true 2024-04-02T14:48:36.219 INFO:teuthology.orchestra.run.smithi169.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-02T14:48:36.219 INFO:teuthology.orchestra.run.smithi169.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-02T14:48:36.219 INFO:teuthology.orchestra.run.smithi169.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-02T14:48:36.219 INFO:teuthology.orchestra.run.smithi169.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-04-02T14:48:36.221 DEBUG:teuthology.orchestra.run.smithi169:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-04-02T14:48:36.391 DEBUG:teuthology.orchestra.run.smithi169:> sudo shutdown -r now 2024-04-02T14:48:36.976 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:48:36.976 INFO:teuthology.orchestra.run.smithi042.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-02T14:48:36.977 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T14:48:37.020 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:48:37.403 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:48:37.403 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-04-02T14:48:37.403 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T14:48:37.403 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T14:48:37.821 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:48:37.822 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T14:48:37.822 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-04-02T14:48:37.822 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-5.14.0-434.el9.x86_64 2024-04-02T14:48:37.822 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-04-02T14:48:37.822 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-04-02T14:48:37.822 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-04-02T14:48:37.822 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T14:48:37.822 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-04-02T14:48:38.147 DEBUG:teuthology.orchestra.run.smithi042:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T14:48:38.255 INFO:teuthology.orchestra.run.smithi042.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T14:48:38.255 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T14:48:38.255 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-04-02T14:48:38.255 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-02T14:48:38.255 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-02T14:48:38.255 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-02T14:48:38.255 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi042.front.sepia.ceph.com, path=None, version=distro) 2024-04-02T14:48:38.255 DEBUG:teuthology.orchestra.run.smithi042:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T14:48:38.339 INFO:teuthology.orchestra.run.smithi042.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T14:48:38.339 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T14:48:38.339 DEBUG:teuthology.orchestra.run.smithi042:> sudo rpm -qi grub2-tools 2024-04-02T14:48:38.386 INFO:teuthology.orchestra.run.smithi042.stdout:Name : grub2-tools 2024-04-02T14:48:38.387 INFO:teuthology.orchestra.run.smithi042.stdout:Epoch : 1 2024-04-02T14:48:38.387 INFO:teuthology.orchestra.run.smithi042.stdout:Version : 2.06 2024-04-02T14:48:38.387 INFO:teuthology.orchestra.run.smithi042.stdout:Release : 68.el9 2024-04-02T14:48:38.387 INFO:teuthology.orchestra.run.smithi042.stdout:Architecture: x86_64 2024-04-02T14:48:38.387 INFO:teuthology.orchestra.run.smithi042.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-02T14:48:38.387 INFO:teuthology.orchestra.run.smithi042.stdout:Group : Unspecified 2024-04-02T14:48:38.387 INFO:teuthology.orchestra.run.smithi042.stdout:Size : 8264254 2024-04-02T14:48:38.387 INFO:teuthology.orchestra.run.smithi042.stdout:License : GPLv3+ 2024-04-02T14:48:38.387 INFO:teuthology.orchestra.run.smithi042.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-02T14:48:38.387 INFO:teuthology.orchestra.run.smithi042.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-02T14:48:38.387 INFO:teuthology.orchestra.run.smithi042.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-02T14:48:38.387 INFO:teuthology.orchestra.run.smithi042.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-02T14:48:38.387 INFO:teuthology.orchestra.run.smithi042.stdout:Packager : builder@centos.org 2024-04-02T14:48:38.388 INFO:teuthology.orchestra.run.smithi042.stdout:Vendor : CentOS 2024-04-02T14:48:38.388 INFO:teuthology.orchestra.run.smithi042.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-02T14:48:38.388 INFO:teuthology.orchestra.run.smithi042.stdout:Summary : Support tools for GRUB. 2024-04-02T14:48:38.388 INFO:teuthology.orchestra.run.smithi042.stdout:Description : 2024-04-02T14:48:38.388 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T14:48:38.388 INFO:teuthology.orchestra.run.smithi042.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-02T14:48:38.388 INFO:teuthology.orchestra.run.smithi042.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-02T14:48:38.388 INFO:teuthology.orchestra.run.smithi042.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-02T14:48:38.388 INFO:teuthology.orchestra.run.smithi042.stdout:hardware devices. 2024-04-02T14:48:38.388 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T14:48:38.388 INFO:teuthology.orchestra.run.smithi042.stdout:This subpackage provides tools for support of all platforms. 2024-04-02T14:48:38.389 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-02T14:48:38.389 INFO:teuthology.task.kernel:Updating grub on smithi042 to boot 5.14.0-434.el9.x86_64 2024-04-02T14:48:38.389 DEBUG:teuthology.orchestra.run.smithi042:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-02T14:48:38.950 INFO:teuthology.orchestra.run.smithi042.stderr:Generating grub configuration file ... 2024-04-02T14:48:40.399 INFO:teuthology.orchestra.run.smithi042.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-02T14:48:40.433 INFO:teuthology.orchestra.run.smithi042.stderr:done 2024-04-02T14:48:40.441 DEBUG:teuthology.orchestra.run.smithi042:> mktemp 2024-04-02T14:48:40.459 INFO:teuthology.orchestra.run.smithi042.stdout:/tmp/tmp.w3zAI4cjfZ 2024-04-02T14:48:40.459 DEBUG:teuthology.orchestra.run.smithi042:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.w3zAI4cjfZ 2024-04-02T14:48:40.532 DEBUG:teuthology.orchestra.run.smithi042:> sudo chmod 0666 /tmp/tmp.w3zAI4cjfZ 2024-04-02T14:48:40.680 DEBUG:teuthology.orchestra.remote:smithi042:/tmp/tmp.w3zAI4cjfZ is 6KB 2024-04-02T14:48:40.730 DEBUG:teuthology.orchestra.run.smithi042:> rm -fr /tmp/tmp.w3zAI4cjfZ 2024-04-02T14:48:40.744 DEBUG:teuthology.orchestra.run.smithi042:> sudo /bin/ls /boot/loader/entries || true 2024-04-02T14:48:40.813 INFO:teuthology.orchestra.run.smithi042.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-02T14:48:40.813 INFO:teuthology.orchestra.run.smithi042.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-02T14:48:40.813 INFO:teuthology.orchestra.run.smithi042.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-02T14:48:40.813 INFO:teuthology.orchestra.run.smithi042.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-04-02T14:48:40.815 DEBUG:teuthology.orchestra.run.smithi042:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-04-02T14:48:40.963 DEBUG:teuthology.orchestra.run.smithi042:> sudo shutdown -r now 2024-04-02T14:49:06.418 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-02T14:49:06.419 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2024-04-02T14:49:06.420 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:49:10.970 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-02T14:49:10.970 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-04-02T14:49:10.971 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi042.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:49:40.899 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.169 2024-04-02T14:49:49.904 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2024-04-02T14:49:49.906 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:50:08.286 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.169 2024-04-02T14:50:10.976 DEBUG:teuthology.orchestra.remote:timed out 2024-04-02T14:50:19.985 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-04-02T14:50:19.986 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi042.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:50:20.278 DEBUG:teuthology.orchestra.run.smithi042:> true 2024-04-02T14:50:20.288 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2024-04-02T14:50:20.288 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:50:20.760 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-04-02T14:50:20.760 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-434.el9.x86_64"... 2024-04-02T14:50:20.761 DEBUG:teuthology.orchestra.run.smithi042:> uname -r 2024-04-02T14:50:20.775 INFO:teuthology.orchestra.run.smithi042.stdout:5.14.0-434.el9.x86_64 2024-04-02T14:50:20.775 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-04-02T14:50:20.775 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-02T14:50:20.775 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-02T14:50:21.776 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-02T14:50:21.776 DEBUG:teuthology.orchestra.run.smithi042:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-02T14:50:21.861 INFO:teuthology.orchestra.run.smithi042.stdout:ttyS1 2024-04-02T14:50:21.900 DEBUG:teuthology.parallel:result is None 2024-04-02T14:50:23.358 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.169 2024-04-02T14:50:38.359 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2024-04-02T14:50:38.360 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:50:38.687 DEBUG:teuthology.orchestra.run.smithi169:> true 2024-04-02T14:50:39.050 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi169.front.sepia.ceph.com' 2024-04-02T14:50:39.050 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-434.el9.x86_64"... 2024-04-02T14:50:39.050 DEBUG:teuthology.orchestra.run.smithi169:> uname -r 2024-04-02T14:50:39.065 INFO:teuthology.orchestra.run.smithi169.stdout:5.14.0-434.el9.x86_64 2024-04-02T14:50:39.066 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-04-02T14:50:39.066 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-02T14:50:39.066 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-02T14:50:40.067 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-02T14:50:40.067 DEBUG:teuthology.orchestra.run.smithi169:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-02T14:50:40.129 INFO:teuthology.orchestra.run.smithi169.stdout:ttyS1 2024-04-02T14:50:40.164 DEBUG:teuthology.parallel:result is None 2024-04-02T14:50:40.164 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-02T14:50:40.176 INFO:teuthology.task.internal:Creating test directory... 2024-04-02T14:50:40.176 DEBUG:teuthology.orchestra.run.smithi042:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-02T14:50:40.178 DEBUG:teuthology.orchestra.run.smithi169:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-02T14:50:40.196 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-02T14:50:40.204 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-02T14:50:40.218 INFO:teuthology.task.internal:Creating archive directory... 2024-04-02T14:50:40.218 DEBUG:teuthology.orchestra.run.smithi042:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-02T14:50:40.237 DEBUG:teuthology.orchestra.run.smithi169:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-02T14:50:40.281 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-02T14:50:40.292 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-02T14:50:40.293 DEBUG:teuthology.orchestra.run.smithi042:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-02T14:50:40.306 DEBUG:teuthology.orchestra.run.smithi169:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-02T14:50:40.337 INFO:teuthology.orchestra.run.smithi042.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T14:50:40.351 INFO:teuthology.orchestra.run.smithi042.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T14:50:40.354 INFO:teuthology.orchestra.run.smithi169.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T14:50:40.368 INFO:teuthology.orchestra.run.smithi169.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T14:50:40.369 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-02T14:50:40.381 INFO:teuthology.task.internal:Configuring sudo... 2024-04-02T14:50:40.381 DEBUG:teuthology.orchestra.run.smithi042:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-02T14:50:40.395 DEBUG:teuthology.orchestra.run.smithi169:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-02T14:50:40.438 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-02T14:50:40.451 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-02T14:50:40.451 DEBUG:teuthology.orchestra.run.smithi042:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-02T14:50:40.465 DEBUG:teuthology.orchestra.run.smithi169:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-02T14:50:40.494 DEBUG:teuthology.orchestra.run.smithi042:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T14:50:40.563 DEBUG:teuthology.orchestra.run.smithi042:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T14:50:40.612 DEBUG:teuthology.orchestra.run.smithi042:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T14:50:40.693 DEBUG:teuthology.orchestra.run.smithi042:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T14:50:40.739 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-04-02T14:50:40.739 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-02T14:50:40.807 DEBUG:teuthology.orchestra.run.smithi169:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T14:50:40.852 DEBUG:teuthology.orchestra.run.smithi169:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T14:50:40.908 DEBUG:teuthology.orchestra.run.smithi169:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T14:50:40.989 DEBUG:teuthology.orchestra.run.smithi169:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T14:50:41.035 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-04-02T14:50:41.035 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-02T14:50:41.104 DEBUG:teuthology.orchestra.run.smithi042:> sudo service rsyslog restart 2024-04-02T14:50:41.106 DEBUG:teuthology.orchestra.run.smithi169:> sudo service rsyslog restart 2024-04-02T14:50:41.153 INFO:teuthology.orchestra.run.smithi042.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T14:50:41.195 INFO:teuthology.orchestra.run.smithi169.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T14:50:41.701 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-02T14:50:41.709 INFO:teuthology.task.internal:Starting timer... 2024-04-02T14:50:41.709 INFO:teuthology.run_tasks:Running task pcp... 2024-04-02T14:50:41.720 INFO:teuthology.run_tasks:Running task selinux... 2024-04-02T14:50:41.730 DEBUG:teuthology.orchestra.run.smithi042:> sudo service auditd rotate 2024-04-02T14:50:41.799 INFO:teuthology.orchestra.run.smithi042.stdout:Rotating logs: 2024-04-02T14:50:41.801 DEBUG:teuthology.orchestra.run.smithi169:> sudo service auditd rotate 2024-04-02T14:50:41.856 INFO:teuthology.orchestra.run.smithi169.stdout:Rotating logs: 2024-04-02T14:50:41.857 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-02T14:50:41.858 DEBUG:teuthology.orchestra.run.smithi042:> /usr/sbin/getenforce 2024-04-02T14:50:41.896 INFO:teuthology.orchestra.run.smithi042.stdout:Permissive 2024-04-02T14:50:41.896 DEBUG:teuthology.orchestra.run.smithi169:> /usr/sbin/getenforce 2024-04-02T14:50:41.937 INFO:teuthology.orchestra.run.smithi169.stdout:Permissive 2024-04-02T14:50:41.937 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi042.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi169.front.sepia.ceph.com': 'permissive'} 2024-04-02T14:50:41.937 DEBUG:teuthology.orchestra.run.smithi042:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-02T14:50:41.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T14:50:41.967 DEBUG:teuthology.orchestra.run.smithi169:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-02T14:50:42.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T14:50:42.005 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-02T14:50:42.005 DEBUG:teuthology.orchestra.run.smithi042:> sudo /usr/sbin/setenforce permissive 2024-04-02T14:50:42.039 DEBUG:teuthology.orchestra.run.smithi169:> sudo /usr/sbin/setenforce permissive 2024-04-02T14:50:42.081 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-02T14:50:42.091 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-02T14:50:42.097 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-02T14:50:42.264 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-02T14:50:42.279 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-02T14:50:42.280 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi042.front.sepia.ceph.com,smithi169.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-02T14:55:03.784 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi042.front.sepia.ceph.com'), Remote(name='ubuntu@smithi169.front.sepia.ceph.com')] 2024-04-02T14:55:03.786 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-04-02T14:55:03.787 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi042.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:55:03.881 DEBUG:teuthology.orchestra.run.smithi042:> true 2024-04-02T14:55:03.957 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-04-02T14:55:03.958 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2024-04-02T14:55:03.958 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:55:04.046 DEBUG:teuthology.orchestra.run.smithi169:> true 2024-04-02T14:55:04.120 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi169.front.sepia.ceph.com' 2024-04-02T14:55:04.121 INFO:teuthology.run_tasks:Running task clock... 2024-04-02T14:55:04.133 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-02T14:55:04.133 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-02T14:55:04.134 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T14:55:04.136 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-02T14:55:04.136 DEBUG:teuthology.orchestra.run.smithi169:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T14:55:04.165 INFO:teuthology.orchestra.run.smithi042.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-02T14:55:04.182 INFO:teuthology.orchestra.run.smithi042.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-02T14:55:04.191 INFO:teuthology.orchestra.run.smithi169.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-02T14:55:04.208 INFO:teuthology.orchestra.run.smithi169.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-02T14:55:04.211 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: ntpd: command not found 2024-04-02T14:55:04.224 INFO:teuthology.orchestra.run.smithi042.stdout:506 Cannot talk to daemon 2024-04-02T14:55:04.237 INFO:teuthology.orchestra.run.smithi169.stderr:sudo: ntpd: command not found 2024-04-02T14:55:04.240 INFO:teuthology.orchestra.run.smithi042.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-02T14:55:04.251 INFO:teuthology.orchestra.run.smithi169.stdout:506 Cannot talk to daemon 2024-04-02T14:55:04.256 INFO:teuthology.orchestra.run.smithi042.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-02T14:55:04.267 INFO:teuthology.orchestra.run.smithi169.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-02T14:55:04.284 INFO:teuthology.orchestra.run.smithi169.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-02T14:55:04.309 INFO:teuthology.orchestra.run.smithi042.stderr:bash: line 1: ntpq: command not found 2024-04-02T14:55:04.313 INFO:teuthology.orchestra.run.smithi042.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T14:55:04.313 INFO:teuthology.orchestra.run.smithi042.stdout:=============================================================================== 2024-04-02T14:55:04.313 INFO:teuthology.orchestra.run.smithi042.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:55:04.313 INFO:teuthology.orchestra.run.smithi042.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:55:04.313 INFO:teuthology.orchestra.run.smithi042.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:55:04.313 INFO:teuthology.orchestra.run.smithi042.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:55:04.338 INFO:teuthology.orchestra.run.smithi169.stderr:bash: line 1: ntpq: command not found 2024-04-02T14:55:04.342 INFO:teuthology.orchestra.run.smithi169.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T14:55:04.343 INFO:teuthology.orchestra.run.smithi169.stdout:=============================================================================== 2024-04-02T14:55:04.343 INFO:teuthology.orchestra.run.smithi169.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:55:04.343 INFO:teuthology.orchestra.run.smithi169.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:55:04.343 INFO:teuthology.orchestra.run.smithi169.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:55:04.343 INFO:teuthology.orchestra.run.smithi169.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:55:04.343 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-02T14:55:04.460 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '3f3f3ed992429b85be729743ad3b39d52dd90744'} 2024-04-02T14:55:04.460 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-02T14:55:04.461 INFO:tasks.cephadm:Cluster fsid is fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:55:04.461 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-02T14:55:04.461 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.42', 'mon.c': '[v2:172.21.15.42:3301,v1:172.21.15.42:6790]', 'mon.b': '172.21.15.169'} 2024-04-02T14:55:04.461 INFO:tasks.cephadm:First mon is mon.a on smithi042 2024-04-02T14:55:04.461 INFO:tasks.cephadm:First mgr is y 2024-04-02T14:55:04.461 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-02T14:55:04.462 DEBUG:teuthology.orchestra.run.smithi042:> sudo hostname $(hostname -s) 2024-04-02T14:55:04.489 DEBUG:teuthology.orchestra.run.smithi169:> sudo hostname $(hostname -s) 2024-04-02T14:55:04.517 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-02T14:55:04.517 DEBUG:teuthology.orchestra.run.smithi042:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-02T14:55:04.797 INFO:teuthology.orchestra.run.smithi042.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 2 14:55 /home/ubuntu/cephtest/cephadm 2024-04-02T14:55:04.797 DEBUG:teuthology.orchestra.run.smithi169:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-02T14:55:05.056 INFO:teuthology.orchestra.run.smithi169.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 2 14:55 /home/ubuntu/cephtest/cephadm 2024-04-02T14:55:05.056 DEBUG:teuthology.orchestra.run.smithi042:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-02T14:55:05.074 DEBUG:teuthology.orchestra.run.smithi169:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-02T14:55:05.101 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-02T14:55:05.102 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-02T14:55:05.117 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-02T14:55:05.948 INFO:teuthology.orchestra.run.smithi042.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-02T14:55:06.037 INFO:teuthology.orchestra.run.smithi169.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-02T14:55:28.825 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-04-02T14:55:28.826 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-02T14:55:28.826 INFO:teuthology.orchestra.run.smithi042.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-02T14:55:28.826 INFO:teuthology.orchestra.run.smithi042.stdout: "repo_digests": [ 2024-04-02T14:55:28.826 INFO:teuthology.orchestra.run.smithi042.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-02T14:55:28.826 INFO:teuthology.orchestra.run.smithi042.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-02T14:55:28.826 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-04-02T14:55:28.826 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-04-02T14:55:31.895 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2024-04-02T14:55:31.895 INFO:teuthology.orchestra.run.smithi169.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-02T14:55:31.895 INFO:teuthology.orchestra.run.smithi169.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-02T14:55:31.895 INFO:teuthology.orchestra.run.smithi169.stdout: "repo_digests": [ 2024-04-02T14:55:31.896 INFO:teuthology.orchestra.run.smithi169.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-02T14:55:31.896 INFO:teuthology.orchestra.run.smithi169.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-02T14:55:31.896 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2024-04-02T14:55:31.896 INFO:teuthology.orchestra.run.smithi169.stdout:} 2024-04-02T14:55:31.917 DEBUG:teuthology.orchestra.run.smithi042:> sudo mkdir -p /etc/ceph 2024-04-02T14:55:31.953 DEBUG:teuthology.orchestra.run.smithi169:> sudo mkdir -p /etc/ceph 2024-04-02T14:55:31.985 DEBUG:teuthology.orchestra.run.smithi042:> sudo chmod 777 /etc/ceph 2024-04-02T14:55:32.024 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod 777 /etc/ceph 2024-04-02T14:55:32.053 INFO:tasks.cephadm:Writing seed config... 2024-04-02T14:55:32.055 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-04-02T14:55:32.055 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-02T14:55:32.055 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-02T14:55:32.055 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-04-02T14:55:32.055 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-02T14:55:32.055 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-02T14:55:32.055 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-02T14:55:32.055 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-02T14:55:32.055 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-02T14:55:32.056 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-04-02T14:55:32.056 DEBUG:teuthology.orchestra.run.smithi042:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-02T14:55:32.082 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = fd27f2e2-f100-11ee-b647-cb9ed24678a4 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-02T14:55:32.082 DEBUG:teuthology.orchestra.run.smithi042:mon.a> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mon.a.service 2024-04-02T14:55:32.124 DEBUG:teuthology.orchestra.run.smithi042:mgr.y> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mgr.y.service 2024-04-02T14:55:32.166 INFO:tasks.cephadm:Bootstrapping... 2024-04-02T14:55:32.167 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.42 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-02T14:55:32.368 INFO:teuthology.orchestra.run.smithi042.stderr:-------------------------------------------------------------------------------- 2024-04-02T14:55:32.369 INFO:teuthology.orchestra.run.smithi042.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'fd27f2e2-f100-11ee-b647-cb9ed24678a4', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.42', '--skip-admin-label'] 2024-04-02T14:55:32.391 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: 4.9.4-dev 2024-04-02T14:55:32.393 INFO:teuthology.orchestra.run.smithi042.stderr:Verifying podman|docker is present... 2024-04-02T14:55:32.414 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: 4.9.4-dev 2024-04-02T14:55:32.416 INFO:teuthology.orchestra.run.smithi042.stderr:Verifying lvm2 is present... 2024-04-02T14:55:32.416 INFO:teuthology.orchestra.run.smithi042.stderr:Verifying time synchronization is in place... 2024-04-02T14:55:32.422 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-02T14:55:32.428 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: inactive 2024-04-02T14:55:32.435 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: enabled 2024-04-02T14:55:32.441 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: active 2024-04-02T14:55:32.441 INFO:teuthology.orchestra.run.smithi042.stderr:Unit chronyd.service is enabled and running 2024-04-02T14:55:32.441 INFO:teuthology.orchestra.run.smithi042.stderr:Repeating the final host check... 2024-04-02T14:55:32.462 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: 4.9.4-dev 2024-04-02T14:55:32.463 INFO:teuthology.orchestra.run.smithi042.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-02T14:55:32.464 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl is present 2024-04-02T14:55:32.464 INFO:teuthology.orchestra.run.smithi042.stderr:lvcreate is present 2024-04-02T14:55:32.469 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-02T14:55:32.475 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: inactive 2024-04-02T14:55:32.481 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: enabled 2024-04-02T14:55:32.487 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: active 2024-04-02T14:55:32.488 INFO:teuthology.orchestra.run.smithi042.stderr:Unit chronyd.service is enabled and running 2024-04-02T14:55:32.488 INFO:teuthology.orchestra.run.smithi042.stderr:Host looks OK 2024-04-02T14:55:32.488 INFO:teuthology.orchestra.run.smithi042.stderr:Cluster fsid: fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:55:32.488 INFO:teuthology.orchestra.run.smithi042.stderr:Acquiring lock 140667753181088 on /run/cephadm/fd27f2e2-f100-11ee-b647-cb9ed24678a4.lock 2024-04-02T14:55:32.489 INFO:teuthology.orchestra.run.smithi042.stderr:Lock 140667753181088 acquired on /run/cephadm/fd27f2e2-f100-11ee-b647-cb9ed24678a4.lock 2024-04-02T14:55:32.489 INFO:teuthology.orchestra.run.smithi042.stderr:Verifying IP 172.21.15.42 port 3300 ... 2024-04-02T14:55:32.489 INFO:teuthology.orchestra.run.smithi042.stderr:Verifying IP 172.21.15.42 port 6789 ... 2024-04-02T14:55:32.489 INFO:teuthology.orchestra.run.smithi042.stderr:Base mon IP is 172.21.15.42, final addrv is [v2:172.21.15.42:3300,v1:172.21.15.42:6789] 2024-04-02T14:55:32.491 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.42 metric 100 2024-04-02T14:55:32.491 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.42 metric 100 2024-04-02T14:55:32.494 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-02T14:55:32.494 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-04-02T14:55:32.494 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-04-02T14:55:32.497 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-02T14:55:32.497 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-02T14:55:32.497 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-02T14:55:32.497 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-04-02T14:55:32.497 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: inet6 fe80::ec4:7aff:febd:1440/64 scope link noprefixroute 2024-04-02T14:55:32.497 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-02T14:55:32.498 INFO:teuthology.orchestra.run.smithi042.stderr:Mon IP `172.21.15.42` is in CIDR network `172.21.0.0/20` 2024-04-02T14:55:32.498 INFO:teuthology.orchestra.run.smithi042.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-02T14:55:32.498 INFO:teuthology.orchestra.run.smithi042.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-02T14:55:32.602 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-02T14:55:32.989 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Getting image source signatures 2024-04-02T14:55:32.989 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-02T14:55:32.989 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-02T14:55:32.989 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-02T14:55:32.989 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-02T14:55:32.990 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-02T14:55:32.990 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-02T14:55:33.145 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Writing manifest to image destination 2024-04-02T14:55:33.297 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-02T14:55:33.687 INFO:teuthology.orchestra.run.smithi042.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-02T14:55:34.229 INFO:teuthology.orchestra.run.smithi042.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-02T14:55:34.229 INFO:teuthology.orchestra.run.smithi042.stderr:Extracting ceph user uid/gid from container image... 2024-04-02T14:55:34.571 INFO:teuthology.orchestra.run.smithi042.stderr:stat: 167 167 2024-04-02T14:55:35.035 INFO:teuthology.orchestra.run.smithi042.stderr:Creating initial keys... 2024-04-02T14:55:35.435 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph-authtool: AQBnHAxmyOeGFRAA3rC0KIGTOX+Nwz9Xoi6geQ== 2024-04-02T14:55:36.348 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph-authtool: AQBoHAxmGNlVEBAAbfORhdB+8e82NYCyZkKYyw== 2024-04-02T14:55:37.279 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph-authtool: AQBpHAxmC1YrDBAAkyyvnaDh1IaN6/dbzg8hWA== 2024-04-02T14:55:37.727 INFO:teuthology.orchestra.run.smithi042.stderr:Creating initial monmap... 2024-04-02T14:55:38.177 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-02T14:55:38.177 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-02T14:55:38.177 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:55:38.177 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-02T14:55:38.649 INFO:teuthology.orchestra.run.smithi042.stderr:monmaptool for a [v2:172.21.15.42:3300,v1:172.21.15.42:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-02T14:55:38.650 INFO:teuthology.orchestra.run.smithi042.stderr:setting min_mon_release = octopus 2024-04-02T14:55:38.650 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/monmaptool: set fsid to fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:55:38.650 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-02T14:55:38.650 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-04-02T14:55:38.650 INFO:teuthology.orchestra.run.smithi042.stderr:Creating mon... 2024-04-02T14:55:39.480 INFO:teuthology.orchestra.run.smithi042.stderr:create mon.a on 2024-04-02T14:55:39.680 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-02T14:55:39.819 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4.target → /etc/systemd/system/ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4.target. 2024-04-02T14:55:39.819 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4.target → /etc/systemd/system/ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4.target. 2024-04-02T14:55:40.111 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Failed to reset failed state of unit ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mon.a.service: Unit ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mon.a.service not loaded. 2024-04-02T14:55:40.119 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Created symlink /etc/systemd/system/ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4.target.wants/ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mon.a.service → /etc/systemd/system/ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@.service. 2024-04-02T14:55:40.773 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:40 smithi042 ceph-mon[24460]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-02T14:55:40.773 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:40 smithi042 bash[24441]: 088fc467283247f3dca5dcbf5ebd3f97638c036496fd92b5341003d6fecad9d8 2024-04-02T14:55:40.785 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: disabled 2024-04-02T14:55:40.792 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: inactive 2024-04-02T14:55:40.792 INFO:teuthology.orchestra.run.smithi042.stderr:firewalld.service is not enabled 2024-04-02T14:55:40.792 INFO:teuthology.orchestra.run.smithi042.stderr:Not possible to enable service . firewalld.service is not available 2024-04-02T14:55:40.793 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for mon to start... 2024-04-02T14:55:40.793 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for mon... 2024-04-02T14:55:41.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:40 smithi042 systemd[1]: Started Ceph mon.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T14:55:41.310 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: cluster: 2024-04-02T14:55:41.310 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: id: fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:55:41.310 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-02T14:55:41.310 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-04-02T14:55:41.310 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: services: 2024-04-02T14:55:41.310 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.598113s) 2024-04-02T14:55:41.310 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-02T14:55:41.311 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-02T14:55:41.311 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-04-02T14:55:41.311 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: data: 2024-04-02T14:55:41.311 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-02T14:55:41.311 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-02T14:55:41.311 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-02T14:55:41.311 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: pgs: 2024-04-02T14:55:41.311 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-04-02T14:55:41.887 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:41 smithi042 ceph-mon[24460]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-02T14:55:41.887 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:41 smithi042 ceph-mon[24460]: monmap e1: 1 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0]} 2024-04-02T14:55:41.887 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:41 smithi042 ceph-mon[24460]: fsmap 2024-04-02T14:55:41.887 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:41 smithi042 ceph-mon[24460]: osdmap e1: 0 total, 0 up, 0 in 2024-04-02T14:55:41.887 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:41 smithi042 ceph-mon[24460]: mgrmap e1: no daemons active 2024-04-02T14:55:41.887 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:41 smithi042 ceph-mon[24460]: from='client.? 172.21.15.42:0/2718431167' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T14:55:41.889 INFO:teuthology.orchestra.run.smithi042.stderr:mon is available 2024-04-02T14:55:41.889 INFO:teuthology.orchestra.run.smithi042.stderr:Assimilating anything we can from ceph.conf... 2024-04-02T14:55:42.358 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-04-02T14:55:42.358 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: [global] 2024-04-02T14:55:42.358 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: fsid = fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:55:42.358 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.42:3300,v1:172.21.15.42:6789] 2024-04-02T14:55:42.358 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-02T14:55:42.358 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-02T14:55:42.358 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-02T14:55:42.358 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-02T14:55:42.358 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-04-02T14:55:42.359 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: [mgr] 2024-04-02T14:55:42.359 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-04-02T14:55:42.359 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-02T14:55:42.359 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-04-02T14:55:42.359 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: [osd] 2024-04-02T14:55:42.359 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-02T14:55:42.359 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-02T14:55:42.984 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:42 smithi042 ceph-mon[24460]: from='client.? 172.21.15.42:0/3307331482' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-02T14:55:42.984 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:42 smithi042 ceph-mon[24460]: from='client.? 172.21.15.42:0/3307331482' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-02T14:55:42.987 INFO:teuthology.orchestra.run.smithi042.stderr:Generating new minimal ceph.conf... 2024-04-02T14:55:43.893 INFO:teuthology.orchestra.run.smithi042.stderr:Restarting the monitor... 2024-04-02T14:55:43.938 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:43 smithi042 ceph-mon[24460]: from='client.? 172.21.15.42:0/2695245925' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:55:43.938 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:43 smithi042 systemd[1]: Stopping Ceph mon.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T14:55:44.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:43 smithi042 bash[24677]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon.a" found: no such container 2024-04-02T14:55:44.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:44 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-a[24456]: 2024-04-02T14:55:44.034+0000 7f2c123b6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T14:55:44.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:44 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-a[24456]: 2024-04-02T14:55:44.034+0000 7f2c123b6700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-02T14:55:44.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:44 smithi042 podman[24691]: 2024-04-02 14:55:44.158840639 +0000 UTC m=+0.202253401 container died 088fc467283247f3dca5dcbf5ebd3f97638c036496fd92b5341003d6fecad9d8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-a, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, vcs-type=git, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_CLEAN=True, version=8, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , release=754) 2024-04-02T14:55:44.668 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:44 smithi042 podman[24691]: 2024-04-02 14:55:44.502433749 +0000 UTC m=+0.545846549 container cleanup 088fc467283247f3dca5dcbf5ebd3f97638c036496fd92b5341003d6fecad9d8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, vendor=Red Hat, Inc., version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, name=centos-stream, com.redhat.component=centos-stream-container, GIT_CLEAN=True) 2024-04-02T14:55:44.669 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:44 smithi042 bash[24691]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-a 2024-04-02T14:55:44.936 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:44 smithi042 podman[24703]: 2024-04-02 14:55:44.678231871 +0000 UTC m=+0.516220119 container remove 088fc467283247f3dca5dcbf5ebd3f97638c036496fd92b5341003d6fecad9d8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-a, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, ceph=True, distribution-scope=public, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, RELEASE=HEAD, version=8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=) 2024-04-02T14:55:44.936 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:44 smithi042 bash[24718]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon.a" found: no such container 2024-04-02T14:55:44.936 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:44 smithi042 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mon.a.service: Deactivated successfully. 2024-04-02T14:55:44.936 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:44 smithi042 systemd[1]: Stopped Ceph mon.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T14:55:44.936 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:44 smithi042 systemd[1]: Starting Ceph mon.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T14:55:45.205 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:44 smithi042 podman[24785]: 2024-04-02 14:55:44.945641322 +0000 UTC m=+0.106359980 container create d1df5488c2195e8e23279790235f6302ddeecd52133da084e02e26feeb77f278 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-a, io.k8s.display-name=CentOS Stream 8, vcs-type=git, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, release=754, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8) 2024-04-02T14:55:45.205 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:44 smithi042 podman[24785]: 2024-04-02 14:55:44.857958407 +0000 UTC m=+0.018677066 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-02T14:55:45.205 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 podman[24785]: 2024-04-02 14:55:45.131174823 +0000 UTC m=+0.291893497 container init d1df5488c2195e8e23279790235f6302ddeecd52133da084e02e26feeb77f278 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-a, name=centos-stream, RELEASE=HEAD, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, io.buildah.version=1.19.8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, version=8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754) 2024-04-02T14:55:45.206 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 podman[24785]: 2024-04-02 14:55:45.134286787 +0000 UTC m=+0.295005448 container start d1df5488c2195e8e23279790235f6302ddeecd52133da084e02e26feeb77f278 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-a, com.redhat.component=centos-stream-container, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vcs-type=git, vendor=Red Hat, Inc., version=8, RELEASE=HEAD, io.openshift.tags=base centos centos-stream) 2024-04-02T14:55:45.206 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: set uid:gid to 167:167 (ceph:ceph) 2024-04-02T14:55:45.206 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-02T14:55:45.206 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: pidfile_write: ignore empty --pid-file 2024-04-02T14:55:45.206 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: load: jerasure load: lrc 2024-04-02T14:55:45.206 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: RocksDB version: 6.15.5 2024-04-02T14:55:45.206 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-02T14:55:45.206 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Compile date Apr 18 2022 2024-04-02T14:55:45.206 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: DB SUMMARY 2024-04-02T14:55:45.206 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: DB Session ID: 1KD7G871N2G0OYAI5UZX 2024-04-02T14:55:45.207 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: CURRENT file: CURRENT 2024-04-02T14:55:45.207 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: IDENTITY file: IDENTITY 2024-04-02T14:55:45.207 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-02T14:55:45.207 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-02T14:55:45.207 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82637 ; 2024-04-02T14:55:45.207 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.error_if_exists: 0 2024-04-02T14:55:45.207 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.create_if_missing: 0 2024-04-02T14:55:45.207 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.paranoid_checks: 1 2024-04-02T14:55:45.207 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-02T14:55:45.207 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.env: 0x557f8ecc6860 2024-04-02T14:55:45.207 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.fs: Posix File System 2024-04-02T14:55:45.207 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.info_log: 0x557f907addc0 2024-04-02T14:55:45.208 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_file_opening_threads: 16 2024-04-02T14:55:45.208 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.statistics: (nil) 2024-04-02T14:55:45.208 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.use_fsync: 0 2024-04-02T14:55:45.208 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_log_file_size: 0 2024-04-02T14:55:45.208 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-02T14:55:45.209 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-02T14:55:45.209 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.keep_log_file_num: 1000 2024-04-02T14:55:45.209 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.recycle_log_file_num: 0 2024-04-02T14:55:45.209 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.allow_fallocate: 1 2024-04-02T14:55:45.209 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.allow_mmap_reads: 0 2024-04-02T14:55:45.209 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.allow_mmap_writes: 0 2024-04-02T14:55:45.209 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.use_direct_reads: 0 2024-04-02T14:55:45.209 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-02T14:55:45.209 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.create_missing_column_families: 0 2024-04-02T14:55:45.209 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.db_log_dir: 2024-04-02T14:55:45.210 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-02T14:55:45.210 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-02T14:55:45.210 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-02T14:55:45.210 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-02T14:55:45.210 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-02T14:55:45.210 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-02T14:55:45.210 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-02T14:55:45.210 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.advise_random_on_open: 1 2024-04-02T14:55:45.210 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.db_write_buffer_size: 0 2024-04-02T14:55:45.210 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.write_buffer_manager: 0x557f9089e2a0 2024-04-02T14:55:45.210 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-02T14:55:45.210 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-02T14:55:45.210 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-02T14:55:45.211 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-02T14:55:45.211 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.rate_limiter: (nil) 2024-04-02T14:55:45.211 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-02T14:55:45.211 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.wal_recovery_mode: 2 2024-04-02T14:55:45.211 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.enable_thread_tracking: 0 2024-04-02T14:55:45.211 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.enable_pipelined_write: 0 2024-04-02T14:55:45.211 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.unordered_write: 0 2024-04-02T14:55:45.211 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-02T14:55:45.211 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-02T14:55:45.211 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-02T14:55:45.211 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-02T14:55:45.211 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.row_cache: None 2024-04-02T14:55:45.211 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.wal_filter: None 2024-04-02T14:55:45.211 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-02T14:55:45.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.allow_ingest_behind: 0 2024-04-02T14:55:45.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.preserve_deletes: 0 2024-04-02T14:55:45.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.two_write_queues: 0 2024-04-02T14:55:45.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.manual_wal_flush: 0 2024-04-02T14:55:45.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.atomic_flush: 0 2024-04-02T14:55:45.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-02T14:55:45.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-02T14:55:45.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-02T14:55:45.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.log_readahead_size: 0 2024-04-02T14:55:45.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-02T14:55:45.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.best_efforts_recovery: 0 2024-04-02T14:55:45.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-02T14:55:45.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.allow_data_in_errors: 0 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.db_host_id: __hostname__ 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_background_jobs: 2 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_background_compactions: -1 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_subcompactions: 1 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_total_wal_size: 0 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_open_files: -1 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.bytes_per_sync: 0 2024-04-02T14:55:45.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-02T14:55:45.214 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-02T14:55:45.214 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compaction_readahead_size: 0 2024-04-02T14:55:45.214 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_background_flushes: -1 2024-04-02T14:55:45.214 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Compression algorithms supported: 2024-04-02T14:55:45.214 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-02T14:55:45.214 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: kZSTD supported: 0 2024-04-02T14:55:45.214 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: kXpressCompression supported: 0 2024-04-02T14:55:45.214 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: kLZ4HCCompression supported: 1 2024-04-02T14:55:45.214 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: kLZ4Compression supported: 1 2024-04-02T14:55:45.214 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: kBZip2Compression supported: 0 2024-04-02T14:55:45.214 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: kZlibCompression supported: 1 2024-04-02T14:55:45.214 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: kSnappyCompression supported: 1 2024-04-02T14:55:45.214 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-02T14:55:45.215 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-02T14:55:45.215 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-02T14:55:45.215 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-02T14:55:45.215 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.merge_operator: 2024-04-02T14:55:45.215 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compaction_filter: None 2024-04-02T14:55:45.215 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compaction_filter_factory: None 2024-04-02T14:55:45.215 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.sst_partitioner_factory: None 2024-04-02T14:55:45.215 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-02T14:55:45.215 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-02T14:55:45.215 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557f90779d00) 2024-04-02T14:55:45.215 INFO:journalctl@ceph.mon.a.smithi042.stdout: cache_index_and_filter_blocks: 1 2024-04-02T14:55:45.215 INFO:journalctl@ceph.mon.a.smithi042.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-02T14:55:45.215 INFO:journalctl@ceph.mon.a.smithi042.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-02T14:55:45.215 INFO:journalctl@ceph.mon.a.smithi042.stdout: pin_top_level_index_and_filter: 1 2024-04-02T14:55:45.216 INFO:journalctl@ceph.mon.a.smithi042.stdout: index_type: 0 2024-04-02T14:55:45.216 INFO:journalctl@ceph.mon.a.smithi042.stdout: data_block_index_type: 0 2024-04-02T14:55:45.216 INFO:journalctl@ceph.mon.a.smithi042.stdout: index_shortening: 1 2024-04-02T14:55:45.216 INFO:journalctl@ceph.mon.a.smithi042.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-02T14:55:45.216 INFO:journalctl@ceph.mon.a.smithi042.stdout: hash_index_allow_collision: 1 2024-04-02T14:55:45.216 INFO:journalctl@ceph.mon.a.smithi042.stdout: checksum: 1 2024-04-02T14:55:45.216 INFO:journalctl@ceph.mon.a.smithi042.stdout: no_block_cache: 0 2024-04-02T14:55:45.216 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_cache: 0x557f907e4170 2024-04-02T14:55:45.216 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_cache_name: BinnedLRUCache 2024-04-02T14:55:45.216 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_cache_options: 2024-04-02T14:55:45.216 INFO:journalctl@ceph.mon.a.smithi042.stdout: capacity : 536870912 2024-04-02T14:55:45.217 INFO:journalctl@ceph.mon.a.smithi042.stdout: num_shard_bits : 4 2024-04-02T14:55:45.217 INFO:journalctl@ceph.mon.a.smithi042.stdout: strict_capacity_limit : 0 2024-04-02T14:55:45.217 INFO:journalctl@ceph.mon.a.smithi042.stdout: high_pri_pool_ratio: 0.000 2024-04-02T14:55:45.217 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_cache_compressed: (nil) 2024-04-02T14:55:45.217 INFO:journalctl@ceph.mon.a.smithi042.stdout: persistent_cache: (nil) 2024-04-02T14:55:45.217 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_size: 4096 2024-04-02T14:55:45.217 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_size_deviation: 10 2024-04-02T14:55:45.217 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_restart_interval: 16 2024-04-02T14:55:45.217 INFO:journalctl@ceph.mon.a.smithi042.stdout: index_block_restart_interval: 1 2024-04-02T14:55:45.217 INFO:journalctl@ceph.mon.a.smithi042.stdout: metadata_block_size: 4096 2024-04-02T14:55:45.217 INFO:journalctl@ceph.mon.a.smithi042.stdout: partition_filters: 0 2024-04-02T14:55:45.218 INFO:journalctl@ceph.mon.a.smithi042.stdout: use_delta_encoding: 1 2024-04-02T14:55:45.218 INFO:journalctl@ceph.mon.a.smithi042.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-02T14:55:45.218 INFO:journalctl@ceph.mon.a.smithi042.stdout: whole_key_filtering: 1 2024-04-02T14:55:45.218 INFO:journalctl@ceph.mon.a.smithi042.stdout: verify_compression: 0 2024-04-02T14:55:45.218 INFO:journalctl@ceph.mon.a.smithi042.stdout: read_amp_bytes_per_bit: 0 2024-04-02T14:55:45.218 INFO:journalctl@ceph.mon.a.smithi042.stdout: format_version: 4 2024-04-02T14:55:45.218 INFO:journalctl@ceph.mon.a.smithi042.stdout: enable_index_compression: 1 2024-04-02T14:55:45.218 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_align: 0 2024-04-02T14:55:45.218 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.write_buffer_size: 33554432 2024-04-02T14:55:45.218 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_write_buffer_number: 2 2024-04-02T14:55:45.218 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compression: NoCompression 2024-04-02T14:55:45.218 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.bottommost_compression: Disabled 2024-04-02T14:55:45.218 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.prefix_extractor: nullptr 2024-04-02T14:55:45.218 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.num_levels: 7 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compression_opts.level: 32767 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compression_opts.strategy: 0 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-02T14:55:45.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-02T14:55:45.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-02T14:55:45.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compression_opts.enabled: false 2024-04-02T14:55:45.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-02T14:55:45.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-02T14:55:45.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-02T14:55:45.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.target_file_size_base: 67108864 2024-04-02T14:55:45.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-02T14:55:45.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-02T14:55:45.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-02T14:55:45.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-02T14:55:45.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-02T14:55:45.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-02T14:55:45.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-02T14:55:45.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-02T14:55:45.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-02T14:55:45.221 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-02T14:55:45.221 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-02T14:55:45.221 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-02T14:55:45.221 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-02T14:55:45.221 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.arena_block_size: 4194304 2024-04-02T14:55:45.221 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-02T14:55:45.221 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-02T14:55:45.221 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-02T14:55:45.221 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.disable_auto_compactions: 0 2024-04-02T14:55:45.221 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-02T14:55:45.221 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-02T14:55:45.221 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-02T14:55:45.222 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-02T14:55:45.222 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-02T14:55:45.222 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-02T14:55:45.222 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-02T14:55:45.222 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-02T14:55:45.222 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-02T14:55:45.222 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-02T14:55:45.222 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.table_properties_collectors: 2024-04-02T14:55:45.222 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.inplace_update_support: 0 2024-04-02T14:55:45.222 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-02T14:55:45.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-02T14:55:45.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-02T14:55:45.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-02T14:55:45.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.bloom_locality: 0 2024-04-02T14:55:45.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.max_successive_merges: 0 2024-04-02T14:55:45.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-02T14:55:45.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.paranoid_file_checks: 0 2024-04-02T14:55:45.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.force_consistency_checks: 1 2024-04-02T14:55:45.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.report_bg_io_stats: 0 2024-04-02T14:55:45.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.ttl: 2592000 2024-04-02T14:55:45.224 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-02T14:55:45.224 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.enable_blob_files: false 2024-04-02T14:55:45.224 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.min_blob_size: 0 2024-04-02T14:55:45.224 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.blob_file_size: 268435456 2024-04-02T14:55:45.224 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-02T14:55:45.224 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-02T14:55:45.224 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-02T14:55:45.224 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-02T14:55:45.224 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-02T14:55:45.224 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-02T14:55:45.224 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712069745159771, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-02T14:55:45.225 INFO:teuthology.orchestra.run.smithi042.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-02T14:55:45.225 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-02T14:55:45.225 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-02T14:55:45.225 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712069745160478, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79187, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77450, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71759, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712069745, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "2f5210f9-1a42-4403-9f8c-02315d58973f", "db_session_id": "1KD7G871N2G0OYAI5UZX"}} 2024-04-02T14:55:45.226 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-02T14:55:45.226 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712069745160812, "job": 1, "event": "recovery_finished"} 2024-04-02T14:55:45.226 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-02T14:55:45.226 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x557f907caa80 2024-04-02T14:55:45.226 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: DB pointer 0x557f907da000 2024-04-02T14:55:45.226 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-02T14:55:45.226 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-02T14:55:45.226 INFO:journalctl@ceph.mon.a.smithi042.stdout: ** DB Stats ** 2024-04-02T14:55:45.226 INFO:journalctl@ceph.mon.a.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T14:55:45.227 INFO:journalctl@ceph.mon.a.smithi042.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-02T14:55:45.227 INFO:journalctl@ceph.mon.a.smithi042.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-02T14:55:45.227 INFO:journalctl@ceph.mon.a.smithi042.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T14:55:45.227 INFO:journalctl@ceph.mon.a.smithi042.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-02T14:55:45.227 INFO:journalctl@ceph.mon.a.smithi042.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-02T14:55:45.227 INFO:journalctl@ceph.mon.a.smithi042.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T14:55:45.227 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-04-02T14:55:45.227 INFO:journalctl@ceph.mon.a.smithi042.stdout: ** Compaction Stats [default] ** 2024-04-02T14:55:45.227 INFO:journalctl@ceph.mon.a.smithi042.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T14:55:45.227 INFO:journalctl@ceph.mon.a.smithi042.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T14:55:45.227 INFO:journalctl@ceph.mon.a.smithi042.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 157.7 0.00 0.00 1 0.000 0 0 2024-04-02T14:55:45.228 INFO:journalctl@ceph.mon.a.smithi042.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 157.7 0.00 0.00 1 0.000 0 0 2024-04-02T14:55:45.228 INFO:journalctl@ceph.mon.a.smithi042.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 157.7 0.00 0.00 1 0.000 0 0 2024-04-02T14:55:45.228 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-04-02T14:55:45.228 INFO:journalctl@ceph.mon.a.smithi042.stdout: ** Compaction Stats [default] ** 2024-04-02T14:55:45.228 INFO:journalctl@ceph.mon.a.smithi042.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T14:55:45.228 INFO:journalctl@ceph.mon.a.smithi042.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T14:55:45.228 INFO:journalctl@ceph.mon.a.smithi042.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 157.7 0.00 0.00 1 0.000 0 0 2024-04-02T14:55:45.228 INFO:journalctl@ceph.mon.a.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T14:55:45.228 INFO:journalctl@ceph.mon.a.smithi042.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T14:55:45.228 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T14:55:45.228 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T14:55:45.228 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T14:55:45.228 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T14:55:45.228 INFO:journalctl@ceph.mon.a.smithi042.stdout: Cumulative compaction: 0.00 GB write, 24.50 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T14:55:45.229 INFO:journalctl@ceph.mon.a.smithi042.stdout: Interval compaction: 0.00 GB write, 24.50 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T14:55:45.229 INFO:journalctl@ceph.mon.a.smithi042.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T14:55:45.229 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-04-02T14:55:45.229 INFO:journalctl@ceph.mon.a.smithi042.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T14:55:45.229 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-04-02T14:55:45.229 INFO:journalctl@ceph.mon.a.smithi042.stdout: ** Compaction Stats [default] ** 2024-04-02T14:55:45.229 INFO:journalctl@ceph.mon.a.smithi042.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T14:55:45.229 INFO:journalctl@ceph.mon.a.smithi042.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T14:55:45.229 INFO:journalctl@ceph.mon.a.smithi042.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 157.7 0.00 0.00 1 0.000 0 0 2024-04-02T14:55:45.229 INFO:journalctl@ceph.mon.a.smithi042.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 157.7 0.00 0.00 1 0.000 0 0 2024-04-02T14:55:45.229 INFO:journalctl@ceph.mon.a.smithi042.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-02T14:55:45.229 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-04-02T14:55:45.229 INFO:journalctl@ceph.mon.a.smithi042.stdout: ** Compaction Stats [default] ** 2024-04-02T14:55:45.229 INFO:journalctl@ceph.mon.a.smithi042.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T14:55:45.230 INFO:journalctl@ceph.mon.a.smithi042.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T14:55:45.230 INFO:journalctl@ceph.mon.a.smithi042.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 157.7 0.00 0.00 1 0.000 0 0 2024-04-02T14:55:45.230 INFO:journalctl@ceph.mon.a.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T14:55:45.230 INFO:journalctl@ceph.mon.a.smithi042.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T14:55:45.230 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T14:55:45.230 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T14:55:45.230 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T14:55:45.230 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T14:55:45.230 INFO:journalctl@ceph.mon.a.smithi042.stdout: Cumulative compaction: 0.00 GB write, 24.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T14:55:45.230 INFO:journalctl@ceph.mon.a.smithi042.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T14:55:45.230 INFO:journalctl@ceph.mon.a.smithi042.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T14:55:45.230 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-04-02T14:55:45.230 INFO:journalctl@ceph.mon.a.smithi042.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T14:55:45.231 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: starting mon.a rank 0 at public addrs [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] at bind addrs [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:55:45.231 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: mon.a@-1(???) e1 preinit fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:55:45.231 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: mon.a@-1(???).mds e1 new map 2024-04-02T14:55:45.231 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: mon.a@-1(???).mds e1 print_map 2024-04-02T14:55:45.231 INFO:journalctl@ceph.mon.a.smithi042.stdout: e1 2024-04-02T14:55:45.231 INFO:journalctl@ceph.mon.a.smithi042.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T14:55:45.231 INFO:journalctl@ceph.mon.a.smithi042.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T14:55:45.231 INFO:journalctl@ceph.mon.a.smithi042.stdout: legacy client fscid: -1 2024-04-02T14:55:45.231 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-04-02T14:55:45.231 INFO:journalctl@ceph.mon.a.smithi042.stdout: No filesystems configured 2024-04-02T14:55:45.231 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-02T14:55:45.231 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T14:55:45.231 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T14:55:45.231 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T14:55:45.232 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-02T14:55:45.232 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 bash[24785]: d1df5488c2195e8e23279790235f6302ddeecd52133da084e02e26feeb77f278 2024-04-02T14:55:45.232 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 systemd[1]: Started Ceph mon.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T14:55:45.232 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-02T14:55:45.232 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: monmap e1: 1 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0]} 2024-04-02T14:55:45.232 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: fsmap 2024-04-02T14:55:45.232 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: osdmap e1: 0 total, 0 up, 0 in 2024-04-02T14:55:45.232 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:45 smithi042 ceph-mon[24805]: mgrmap e1: no daemons active 2024-04-02T14:55:46.314 INFO:teuthology.orchestra.run.smithi042.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-02T14:55:46.314 INFO:teuthology.orchestra.run.smithi042.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-02T14:55:46.314 INFO:teuthology.orchestra.run.smithi042.stderr:Creating mgr... 2024-04-02T14:55:46.315 INFO:teuthology.orchestra.run.smithi042.stderr:Verifying port 9283 ... 2024-04-02T14:55:46.483 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Failed to reset failed state of unit ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mgr.y.service: Unit ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mgr.y.service not loaded. 2024-04-02T14:55:46.490 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Created symlink /etc/systemd/system/ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4.target.wants/ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mgr.y.service → /etc/systemd/system/ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@.service. 2024-04-02T14:55:46.704 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:46 smithi042 systemd[1]: Starting Ceph mgr.y for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T14:55:46.959 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:46 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1056356352' entity='client.admin' 2024-04-02T14:55:46.960 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:46 smithi042 podman[25033]: 2024-04-02 14:55:46.863366947 +0000 UTC m=+0.105160228 container create 92a15ec7fb5f99b61368b9326d37fdd66955da4d42d1f6254040003f84d434b6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_CLEAN=True, ceph=True, com.redhat.component=centos-stream-container, release=754, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.openshift.expose-services=, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, build-date=2022-05-03T08:36:31.336870) 2024-04-02T14:55:46.960 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:46 smithi042 podman[25033]: 2024-04-02 14:55:46.776354234 +0000 UTC m=+0.018147470 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-02T14:55:47.295 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:47 smithi042 podman[25033]: 2024-04-02 14:55:47.06615268 +0000 UTC m=+0.307945918 container init 92a15ec7fb5f99b61368b9326d37fdd66955da4d42d1f6254040003f84d434b6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, GIT_BRANCH=HEAD, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , distribution-scope=public, version=8, release=754, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T14:55:47.295 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:47 smithi042 podman[25033]: 2024-04-02 14:55:47.068705899 +0000 UTC m=+0.310499134 container start 92a15ec7fb5f99b61368b9326d37fdd66955da4d42d1f6254040003f84d434b6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, name=centos-stream, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, version=8, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, architecture=x86_64, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T14:55:47.715 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: disabled 2024-04-02T14:55:47.723 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: inactive 2024-04-02T14:55:47.723 INFO:teuthology.orchestra.run.smithi042.stderr:firewalld.service is not enabled 2024-04-02T14:55:47.724 INFO:teuthology.orchestra.run.smithi042.stderr:Not possible to enable service . firewalld.service is not available 2024-04-02T14:55:47.732 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: disabled 2024-04-02T14:55:47.740 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: inactive 2024-04-02T14:55:47.741 INFO:teuthology.orchestra.run.smithi042.stderr:firewalld.service is not enabled 2024-04-02T14:55:47.742 INFO:teuthology.orchestra.run.smithi042.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-02T14:55:47.742 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for mgr to start... 2024-04-02T14:55:47.742 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for mgr... 2024-04-02T14:55:47.867 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:47 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:47.557+0000 7f548421e000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T14:55:47.868 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:47 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:47.647+0000 7f548421e000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T14:55:47.868 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:47 smithi042 bash[25033]: 92a15ec7fb5f99b61368b9326d37fdd66955da4d42d1f6254040003f84d434b6 2024-04-02T14:55:47.868 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:47 smithi042 systemd[1]: Started Ceph mgr.y for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T14:55:48.545 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:48 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:48.273+0000 7f548421e000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T14:55:48.545 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:48 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:48.365+0000 7f548421e000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T14:55:48.545 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:48 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:48.450+0000 7f548421e000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T14:55:48.891 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:48 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:48.600+0000 7f548421e000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T14:55:48.891 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:48 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:48.682+0000 7f548421e000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T14:55:48.891 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:48 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:48.796+0000 7f548421e000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T14:55:48.891 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:48 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:48.885+0000 7f548421e000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T14:55:49.295 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:49 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:49.051+0000 7f548421e000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T14:55:49.295 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:49 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:49.138+0000 7f548421e000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T14:55:50.513 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:50 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:50.198+0000 7f548421e000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T14:55:50.514 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:50 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:50.329+0000 7f548421e000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T14:55:50.514 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:50 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:50.521+0000 7f548421e000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T14:55:50.772 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:50 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:50.610+0000 7f548421e000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T14:55:50.772 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:50 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:50.761+0000 7f548421e000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T14:55:51.367 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:51 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/4006033575' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T14:55:51.367 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:51.087+0000 7f548421e000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T14:55:51.489 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-04-02T14:55:51.489 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: { 2024-04-02T14:55:51.489 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "fsid": "fd27f2e2-f100-11ee-b647-cb9ed24678a4", 2024-04-02T14:55:51.490 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "health": { 2024-04-02T14:55:51.490 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-02T14:55:51.490 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "checks": {}, 2024-04-02T14:55:51.490 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mutes": [] 2024-04-02T14:55:51.490 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-04-02T14:55:51.490 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-02T14:55:51.490 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum": [ 2024-04-02T14:55:51.490 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 0 2024-04-02T14:55:51.490 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-04-02T14:55:51.490 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-02T14:55:51.490 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "a" 2024-04-02T14:55:51.490 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-04-02T14:55:51.490 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-04-02T14:55:51.490 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "monmap": { 2024-04-02T14:55:51.490 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:55:51.491 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-02T14:55:51.491 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-02T14:55:51.491 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-04-02T14:55:51.491 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osdmap": { 2024-04-02T14:55:51.491 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:55:51.491 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-02T14:55:51.491 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-02T14:55:51.491 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-02T14:55:51.491 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-02T14:55:51.491 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-02T14:55:51.491 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-02T14:55:51.491 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-04-02T14:55:51.491 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "pgmap": { 2024-04-02T14:55:51.491 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-02T14:55:51.491 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-02T14:55:51.492 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-02T14:55:51.492 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-02T14:55:51.493 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-02T14:55:51.493 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-02T14:55:51.493 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-02T14:55:51.493 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-02T14:55:51.493 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-04-02T14:55:51.494 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "fsmap": { 2024-04-02T14:55:51.494 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:55:51.494 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-02T14:55:51.494 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-02T14:55:51.494 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-04-02T14:55:51.494 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-02T14:55:51.494 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "available": false, 2024-04-02T14:55:51.494 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-02T14:55:51.494 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "modules": [ 2024-04-02T14:55:51.494 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "iostat", 2024-04-02T14:55:51.494 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "nfs", 2024-04-02T14:55:51.494 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "restful" 2024-04-02T14:55:51.494 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-04-02T14:55:51.495 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "services": {} 2024-04-02T14:55:51.495 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-04-02T14:55:51.495 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "servicemap": { 2024-04-02T14:55:51.495 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:55:51.495 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "modified": "2024-04-02T14:55:40.720578+0000", 2024-04-02T14:55:51.495 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "services": {} 2024-04-02T14:55:51.495 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-04-02T14:55:51.495 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-02T14:55:51.495 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: } 2024-04-02T14:55:51.665 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:51.310+0000 7f548421e000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T14:55:51.665 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:51.394+0000 7f548421e000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T14:55:51.665 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:51.568+0000 7f548421e000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T14:55:52.785 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:52 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:52.497+0000 7f548421e000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T14:55:52.785 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:52 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:52.794+0000 7f548421e000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T14:55:53.045 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:52 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:52.888+0000 7f548421e000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T14:55:53.045 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:55:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:55:53.039+0000 7f548421e000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T14:55:53.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:53 smithi042 ceph-mon[24805]: Activating manager daemon y 2024-04-02T14:55:53.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:53 smithi042 ceph-mon[24805]: mgrmap e2: y(active, starting, since 0.0027127s) 2024-04-02T14:55:53.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:53 smithi042 ceph-mon[24805]: from='mgr.14100 172.21.15.42:0/4204287238' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T14:55:53.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:53 smithi042 ceph-mon[24805]: from='mgr.14100 172.21.15.42:0/4204287238' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T14:55:53.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:53 smithi042 ceph-mon[24805]: from='mgr.14100 172.21.15.42:0/4204287238' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T14:55:53.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:53 smithi042 ceph-mon[24805]: from='mgr.14100 172.21.15.42:0/4204287238' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:55:53.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:53 smithi042 ceph-mon[24805]: from='mgr.14100 172.21.15.42:0/4204287238' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T14:55:53.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:53 smithi042 ceph-mon[24805]: Manager daemon y is now available 2024-04-02T14:55:53.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:53 smithi042 ceph-mon[24805]: from='mgr.14100 172.21.15.42:0/4204287238' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:55:53.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:53 smithi042 ceph-mon[24805]: from='mgr.14100 172.21.15.42:0/4204287238' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T14:55:53.546 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:53 smithi042 ceph-mon[24805]: from='mgr.14100 172.21.15.42:0/4204287238' entity='mgr.y' 2024-04-02T14:55:53.546 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:53 smithi042 ceph-mon[24805]: from='mgr.14100 172.21.15.42:0/4204287238' entity='mgr.y' 2024-04-02T14:55:53.546 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:53 smithi042 ceph-mon[24805]: from='mgr.14100 172.21.15.42:0/4204287238' entity='mgr.y' 2024-04-02T14:55:55.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:55 smithi042 ceph-mon[24805]: mgrmap e3: y(active, since 1.00613s) 2024-04-02T14:55:56.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:56 smithi042 ceph-mon[24805]: mgrmap e4: y(active, since 2s) 2024-04-02T14:55:57.191 INFO:teuthology.orchestra.run.smithi042.stderr:mgr not available, waiting (1/15)... 2024-04-02T14:55:59.904 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-04-02T14:55:59.904 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: { 2024-04-02T14:55:59.904 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "fsid": "fd27f2e2-f100-11ee-b647-cb9ed24678a4", 2024-04-02T14:55:59.904 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "health": { 2024-04-02T14:55:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-02T14:55:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "checks": {}, 2024-04-02T14:55:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mutes": [] 2024-04-02T14:55:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-04-02T14:55:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-02T14:55:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum": [ 2024-04-02T14:55:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 0 2024-04-02T14:55:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-04-02T14:55:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-02T14:55:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "a" 2024-04-02T14:55:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-04-02T14:55:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum_age": 14, 2024-04-02T14:55:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "monmap": { 2024-04-02T14:55:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:55:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-02T14:55:59.906 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-02T14:55:59.906 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-04-02T14:55:59.906 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osdmap": { 2024-04-02T14:55:59.906 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:55:59.906 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-02T14:55:59.906 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-02T14:55:59.906 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-02T14:55:59.906 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-02T14:55:59.906 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-02T14:55:59.906 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-02T14:55:59.906 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-04-02T14:55:59.906 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "pgmap": { 2024-04-02T14:55:59.906 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-02T14:55:59.906 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-02T14:55:59.906 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-02T14:55:59.907 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-02T14:55:59.907 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-02T14:55:59.907 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-02T14:55:59.907 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-02T14:55:59.907 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-02T14:55:59.907 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-04-02T14:55:59.907 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "fsmap": { 2024-04-02T14:55:59.907 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:55:59.907 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-02T14:55:59.907 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-02T14:55:59.907 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-04-02T14:55:59.907 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-02T14:55:59.907 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "available": true, 2024-04-02T14:55:59.907 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-02T14:55:59.907 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "modules": [ 2024-04-02T14:55:59.908 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "iostat", 2024-04-02T14:55:59.908 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "nfs", 2024-04-02T14:55:59.908 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "restful" 2024-04-02T14:55:59.908 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-04-02T14:55:59.908 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "services": {} 2024-04-02T14:55:59.908 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-04-02T14:55:59.908 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "servicemap": { 2024-04-02T14:55:59.908 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:55:59.908 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "modified": "2024-04-02T14:55:40.720578+0000", 2024-04-02T14:55:59.908 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "services": {} 2024-04-02T14:55:59.908 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-04-02T14:55:59.908 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-02T14:55:59.908 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: } 2024-04-02T14:56:00.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:55:59 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/667519953' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T14:56:00.468 INFO:teuthology.orchestra.run.smithi042.stderr:mgr is available 2024-04-02T14:56:01.033 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-04-02T14:56:01.033 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: [global] 2024-04-02T14:56:01.033 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: fsid = fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:56:01.033 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-02T14:56:01.033 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-02T14:56:01.034 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-02T14:56:01.034 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-02T14:56:01.034 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-04-02T14:56:01.034 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: [mgr] 2024-04-02T14:56:01.034 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-02T14:56:01.034 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-04-02T14:56:01.034 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: [osd] 2024-04-02T14:56:01.034 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-02T14:56:01.034 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-02T14:56:01.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:01 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3965186407' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-02T14:56:01.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:01 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3965186407' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-02T14:56:01.634 INFO:teuthology.orchestra.run.smithi042.stderr:Enabling cephadm module... 2024-04-02T14:56:02.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:02 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1810667459' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-02T14:56:02.795 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:02 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ignoring --setuser ceph since I am not root 2024-04-02T14:56:02.795 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:02 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ignoring --setgroup ceph since I am not root 2024-04-02T14:56:03.295 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:02 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:02.808+0000 7fc47d541000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T14:56:03.295 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:02 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:02.900+0000 7fc47d541000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T14:56:03.588 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:03 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1810667459' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-02T14:56:03.588 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:03 smithi042 ceph-mon[24805]: mgrmap e5: y(active, since 9s) 2024-04-02T14:56:03.588 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:03.506+0000 7fc47d541000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T14:56:03.588 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:03.597+0000 7fc47d541000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T14:56:03.860 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:03.685+0000 7fc47d541000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T14:56:03.860 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:03.834+0000 7fc47d541000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T14:56:04.126 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:03.919+0000 7fc47d541000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T14:56:04.126 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:04 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:04.030+0000 7fc47d541000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T14:56:04.126 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:04 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:04.135+0000 7fc47d541000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T14:56:04.321 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: { 2024-04-02T14:56:04.321 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-02T14:56:04.321 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "available": true, 2024-04-02T14:56:04.321 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-02T14:56:04.321 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-02T14:56:04.322 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: } 2024-04-02T14:56:04.379 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:04 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3858229135' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-02T14:56:04.379 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:04 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:04.298+0000 7fc47d541000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T14:56:04.380 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:04 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:04.388+0000 7fc47d541000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T14:56:04.884 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for the mgr to restart... 2024-04-02T14:56:04.885 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for mgr epoch 5... 2024-04-02T14:56:05.750 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:05 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:05.436+0000 7fc47d541000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T14:56:05.750 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:05 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:05.567+0000 7fc47d541000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T14:56:05.750 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:05 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:05.759+0000 7fc47d541000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T14:56:06.044 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:05 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:05.847+0000 7fc47d541000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T14:56:06.045 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:05 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:05.984+0000 7fc47d541000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T14:56:06.545 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:06 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:06.303+0000 7fc47d541000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T14:56:06.545 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:06 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:06.526+0000 7fc47d541000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T14:56:07.044 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:06 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:06.611+0000 7fc47d541000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T14:56:07.045 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:06 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:06.784+0000 7fc47d541000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T14:56:08.010 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:07 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:07.720+0000 7fc47d541000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T14:56:08.011 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:08 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:08.019+0000 7fc47d541000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T14:56:08.295 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:08 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:08.114+0000 7fc47d541000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T14:56:08.295 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:08 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:08.267+0000 7fc47d541000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T14:56:09.321 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: { 2024-04-02T14:56:09.321 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-02T14:56:09.321 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "initialized": true 2024-04-02T14:56:09.321 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: } 2024-04-02T14:56:09.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: Active manager daemon y restarted 2024-04-02T14:56:09.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: Activating manager daemon y 2024-04-02T14:56:09.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: osdmap e2: 0 total, 0 up, 0 in 2024-04-02T14:56:09.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: mgrmap e6: y(active, starting, since 0.0527347s) 2024-04-02T14:56:09.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:56:09.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T14:56:09.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T14:56:09.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T14:56:09.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T14:56:09.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: Manager daemon y is now available 2024-04-02T14:56:09.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' 2024-04-02T14:56:09.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' 2024-04-02T14:56:09.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:09.546 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:09.546 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:09.546 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:56:09.546 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:09 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T14:56:09.873 INFO:teuthology.orchestra.run.smithi042.stderr:mgr epoch 5 is available 2024-04-02T14:56:09.873 INFO:teuthology.orchestra.run.smithi042.stderr:Setting orchestrator backend to cephadm... 2024-04-02T14:56:10.472 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:10 smithi042 ceph-mon[24805]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-02T14:56:10.472 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:10 smithi042 ceph-mon[24805]: mgrmap e7: y(active, since 1.0549s) 2024-04-02T14:56:10.472 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:10 smithi042 ceph-mon[24805]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-02T14:56:11.606 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: value unchanged 2024-04-02T14:56:11.687 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:11 smithi042 ceph-mon[24805]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:11.688 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:11 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' 2024-04-02T14:56:11.688 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:11 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:11.688 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:11 smithi042 ceph-mon[24805]: mgrmap e8: y(active, since 2s) 2024-04-02T14:56:11.688 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: [02/Apr/2024:14:56:11] ENGINE Bus STARTING 2024-04-02T14:56:11.688 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: [02/Apr/2024:14:56:11] ENGINE Serving on https://172.21.15.42:7150 2024-04-02T14:56:11.950 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: [02/Apr/2024:14:56:11] ENGINE Bus STARTED 2024-04-02T14:56:12.194 INFO:teuthology.orchestra.run.smithi042.stderr:Generating ssh key... 2024-04-02T14:56:13.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:12 smithi042 ceph-mon[24805]: [02/Apr/2024:14:56:11] ENGINE Bus STARTING 2024-04-02T14:56:13.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:12 smithi042 ceph-mon[24805]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:13.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:12 smithi042 ceph-mon[24805]: [02/Apr/2024:14:56:11] ENGINE Serving on https://172.21.15.42:7150 2024-04-02T14:56:13.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:12 smithi042 ceph-mon[24805]: [02/Apr/2024:14:56:11] ENGINE Bus STARTED 2024-04-02T14:56:13.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:12 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' 2024-04-02T14:56:13.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:12 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:13.382 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: Generating public/private rsa key pair. 2024-04-02T14:56:13.382 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: Your identification has been saved in /tmp/tmpwhgie6_c/key. 2024-04-02T14:56:13.382 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: Your public key has been saved in /tmp/tmpwhgie6_c/key.pub. 2024-04-02T14:56:13.382 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: The key fingerprint is: 2024-04-02T14:56:13.382 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: SHA256:EfSeswVBvxarYHX6MawxXM10k4OKxN44d3J1d7Gj/68 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:56:13.383 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: The key's randomart image is: 2024-04-02T14:56:13.383 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: +---[RSA 3072]----+ 2024-04-02T14:56:13.383 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: | .+.o oo+| 2024-04-02T14:56:13.383 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: | = o = =*| 2024-04-02T14:56:13.383 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: | + B * +o=| 2024-04-02T14:56:13.383 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: | X % *. .| 2024-04-02T14:56:13.383 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: | S @ %. | 2024-04-02T14:56:13.383 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: | . . @ o. | 2024-04-02T14:56:13.383 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: | + . . | 2024-04-02T14:56:13.383 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: | .| 2024-04-02T14:56:13.383 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: | E.+| 2024-04-02T14:56:13.383 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: +----[SHA256]-----+ 2024-04-02T14:56:14.195 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:14 smithi042 ceph-mon[24805]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:14.195 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:14 smithi042 ceph-mon[24805]: Generating ssh key... 2024-04-02T14:56:14.195 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:14 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' 2024-04-02T14:56:14.196 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:14 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' 2024-04-02T14:56:14.196 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD5Dvxf/nkW3/W3wJatdfdBD+WE3xgdsCTDIyX2eUuUsaPIpk/kHPyB9O1+lNCK1eHLxkgDOMxPacbedm6MXuru/XAe9VNz0GQ6D9Ev0Lf9DSxTkfoGKdwLVdRbxrgYNVbX/5Zxo6wrGvxSEmYXBb4G2VHpdT7vIk/F+RJNaripCD8jMiOKXvCUis1iSLWtaWyU09GMTFirvFmIE6zgzLXpCpkPhFWP69IvVWxxD9liSq2YHrNe2q3LX3+TbW3cNxJZib99wSeaS1Z1XN+pkAzrNxnkKEKdwk4LCjlVYYjXH5JasXXUW/59/bT3NphUw8mqc472di+/4PjPO/0Eso8s7KhlMyqqwhJlaY2fSY8VbsMPxsTWCBVAwayf8D0oAYFHy7KxPc0N+glTAT7Gv13y2T/ogOwxsAzuXPSFaUdyxdh+PYH4NtJz9YjI7IQZHpp3l+3i3SjVf/thXvynLjA6TqPxFcVHgq5Pe/KD1mKJv0ElAmV1kVQY7PGM+sVZQHU= ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:56:14.815 INFO:teuthology.orchestra.run.smithi042.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-02T14:56:14.816 INFO:teuthology.orchestra.run.smithi042.stderr:Adding key to root@localhost authorized_keys... 2024-04-02T14:56:14.816 INFO:teuthology.orchestra.run.smithi042.stderr:Adding host smithi042... 2024-04-02T14:56:15.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:15 smithi042 ceph-mon[24805]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:16.381 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: Added host 'smithi042' with addr '172.21.15.42' 2024-04-02T14:56:16.712 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:16 smithi042 ceph-mon[24805]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi042", "addr": "172.21.15.42", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:16.712 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:16 smithi042 ceph-mon[24805]: Deploying cephadm binary to smithi042 2024-04-02T14:56:16.712 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:16 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' 2024-04-02T14:56:16.973 INFO:teuthology.orchestra.run.smithi042.stderr:Deploying unmanaged mon service... 2024-04-02T14:56:17.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:17 smithi042 ceph-mon[24805]: Added host smithi042 2024-04-02T14:56:17.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:17 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:17.600 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-02T14:56:18.298 INFO:teuthology.orchestra.run.smithi042.stderr:Deploying unmanaged mgr service... 2024-04-02T14:56:18.882 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:18 smithi042 ceph-mon[24805]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:18.882 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:18 smithi042 ceph-mon[24805]: Saving service mon spec with placement count:5 2024-04-02T14:56:18.882 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:18 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' 2024-04-02T14:56:19.855 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-02T14:56:20.744 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:20 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' 2024-04-02T14:56:20.744 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:20 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' 2024-04-02T14:56:20.744 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:20 smithi042 ceph-mon[24805]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:20.745 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:20 smithi042 ceph-mon[24805]: Saving service mgr spec with placement count:2 2024-04-02T14:56:20.745 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:20 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' 2024-04-02T14:56:22.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:22 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1745649' entity='client.admin' 2024-04-02T14:56:23.067 INFO:teuthology.orchestra.run.smithi042.stderr:Enabling the dashboard module... 2024-04-02T14:56:23.485 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:23 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' 2024-04-02T14:56:23.485 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:23 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/172597757' entity='client.admin' 2024-04-02T14:56:25.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:24 smithi042 ceph-mon[24805]: from='mgr.14116 172.21.15.42:0/765508494' entity='mgr.y' 2024-04-02T14:56:25.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:24 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3580043627' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-02T14:56:25.045 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:24 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ignoring --setuser ceph since I am not root 2024-04-02T14:56:25.045 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:24 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ignoring --setgroup ceph since I am not root 2024-04-02T14:56:25.470 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:25 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:25.097+0000 7ff21556c000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T14:56:25.470 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:25 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:25.190+0000 7ff21556c000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T14:56:25.720 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:25 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3580043627' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-02T14:56:25.720 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:25 smithi042 ceph-mon[24805]: mgrmap e9: y(active, since 16s) 2024-04-02T14:56:25.980 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:25 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:25.807+0000 7ff21556c000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T14:56:25.981 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:25 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:25.899+0000 7ff21556c000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T14:56:25.981 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:25 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:25.990+0000 7ff21556c000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T14:56:26.147 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: { 2024-04-02T14:56:26.147 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-02T14:56:26.147 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "available": true, 2024-04-02T14:56:26.147 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-02T14:56:26.147 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-02T14:56:26.147 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: } 2024-04-02T14:56:26.231 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:26 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:26.141+0000 7ff21556c000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T14:56:26.231 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:26 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:26.234+0000 7ff21556c000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T14:56:26.546 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:26 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:26.347+0000 7ff21556c000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T14:56:26.547 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:26 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:26.432+0000 7ff21556c000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T14:56:26.654 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for the mgr to restart... 2024-04-02T14:56:26.654 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for mgr epoch 9... 2024-04-02T14:56:26.926 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:26 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1546011662' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-02T14:56:26.926 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:26 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:26.598+0000 7ff21556c000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T14:56:26.926 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:26 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:26.688+0000 7ff21556c000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T14:56:28.044 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:27 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:27.738+0000 7ff21556c000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T14:56:28.045 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:27 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:27.870+0000 7ff21556c000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T14:56:28.544 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:28.064+0000 7ff21556c000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T14:56:28.545 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:28.152+0000 7ff21556c000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T14:56:28.545 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:28.291+0000 7ff21556c000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T14:56:28.914 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:28.612+0000 7ff21556c000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T14:56:28.915 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:28.838+0000 7ff21556c000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T14:56:28.915 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:28.923+0000 7ff21556c000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T14:56:29.294 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:29 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:29.097+0000 7ff21556c000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T14:56:30.294 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:30 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:30.031+0000 7ff21556c000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T14:56:30.794 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:30 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:30.334+0000 7ff21556c000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T14:56:30.795 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:30 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:30.430+0000 7ff21556c000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T14:56:30.795 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:30 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T14:56:30.582+0000 7ff21556c000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T14:56:31.875 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: [02/Apr/2024:14:56:31] ENGINE Bus STARTING 2024-04-02T14:56:31.875 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: [02/Apr/2024:14:56:31] ENGINE Serving on https://172.21.15.42:7150 2024-04-02T14:56:31.875 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: [02/Apr/2024:14:56:31] ENGINE Bus STARTED 2024-04-02T14:56:31.875 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-mon[24805]: Active manager daemon y restarted 2024-04-02T14:56:31.875 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-mon[24805]: Activating manager daemon y 2024-04-02T14:56:31.875 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-mon[24805]: osdmap e3: 0 total, 0 up, 0 in 2024-04-02T14:56:31.876 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-mon[24805]: mgrmap e10: y(active, starting, since 0.590284s) 2024-04-02T14:56:31.876 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:56:31.876 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T14:56:31.876 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T14:56:31.876 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T14:56:31.876 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T14:56:31.876 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-mon[24805]: Manager daemon y is now available 2024-04-02T14:56:31.876 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:31.876 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:31.876 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:56:31.876 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T14:56:32.174 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: { 2024-04-02T14:56:32.174 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-02T14:56:32.175 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "initialized": true 2024-04-02T14:56:32.175 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: } 2024-04-02T14:56:32.937 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:32 smithi042 ceph-mon[24805]: [02/Apr/2024:14:56:31] ENGINE Bus STARTING 2024-04-02T14:56:32.937 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:32 smithi042 ceph-mon[24805]: [02/Apr/2024:14:56:31] ENGINE Serving on https://172.21.15.42:7150 2024-04-02T14:56:32.937 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:32 smithi042 ceph-mon[24805]: [02/Apr/2024:14:56:31] ENGINE Bus STARTED 2024-04-02T14:56:32.937 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:32 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:32.937 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:32 smithi042 ceph-mon[24805]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-02T14:56:32.937 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:32 smithi042 ceph-mon[24805]: mgrmap e11: y(active, since 1.59249s) 2024-04-02T14:56:32.937 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:32 smithi042 ceph-mon[24805]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-02T14:56:32.940 INFO:teuthology.orchestra.run.smithi042.stderr:mgr epoch 9 is available 2024-04-02T14:56:32.940 INFO:teuthology.orchestra.run.smithi042.stderr:Generating a dashboard self-signed certificate... 2024-04-02T14:56:33.829 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-02T14:56:34.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:34.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:34 smithi042 ceph-mon[24805]: mgrmap e12: y(active, since 2s) 2024-04-02T14:56:34.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:34 smithi042 ceph-mon[24805]: from='client.14160 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:34.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:34.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:34.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:34.454 INFO:teuthology.orchestra.run.smithi042.stderr:Creating initial admin user... 2024-04-02T14:56:35.472 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$pWtJQp06HWUYRbkMhyyz.eD7kWVRrYysxSKN3WmL6TfAcNgnhgrne", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1712069795, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-02T14:56:36.607 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:36 smithi042 ceph-mon[24805]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:36.607 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:36 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:36.609 INFO:teuthology.orchestra.run.smithi042.stderr:Fetching dashboard port number... 2024-04-02T14:56:37.278 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 8443 2024-04-02T14:56:37.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:37 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3856630286' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-02T14:56:37.925 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: disabled 2024-04-02T14:56:37.931 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: inactive 2024-04-02T14:56:37.932 INFO:teuthology.orchestra.run.smithi042.stderr:firewalld.service is not enabled 2024-04-02T14:56:37.932 INFO:teuthology.orchestra.run.smithi042.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-02T14:56:37.934 INFO:teuthology.orchestra.run.smithi042.stderr:Ceph Dashboard is now available at: 2024-04-02T14:56:37.934 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-04-02T14:56:37.934 INFO:teuthology.orchestra.run.smithi042.stderr: URL: https://smithi042.front.sepia.ceph.com:8443/ 2024-04-02T14:56:37.934 INFO:teuthology.orchestra.run.smithi042.stderr: User: admin 2024-04-02T14:56:37.934 INFO:teuthology.orchestra.run.smithi042.stderr: Password: tjsfamam59 2024-04-02T14:56:37.934 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-04-02T14:56:37.934 INFO:teuthology.orchestra.run.smithi042.stderr:Enabling autotune for osd_memory_target 2024-04-02T14:56:38.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:38 smithi042 ceph-mon[24805]: mgrmap e13: y(active, since 6s) 2024-04-02T14:56:39.888 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-02T14:56:40.513 INFO:teuthology.orchestra.run.smithi042.stderr:You can access the Ceph CLI with: 2024-04-02T14:56:40.513 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-04-02T14:56:40.513 INFO:teuthology.orchestra.run.smithi042.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-02T14:56:40.513 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-04-02T14:56:40.513 INFO:teuthology.orchestra.run.smithi042.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-02T14:56:40.513 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-04-02T14:56:40.513 INFO:teuthology.orchestra.run.smithi042.stderr: ceph telemetry on 2024-04-02T14:56:40.513 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-04-02T14:56:40.513 INFO:teuthology.orchestra.run.smithi042.stderr:For more information see: 2024-04-02T14:56:40.514 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-04-02T14:56:40.514 INFO:teuthology.orchestra.run.smithi042.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-02T14:56:40.514 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-04-02T14:56:40.514 INFO:teuthology.orchestra.run.smithi042.stderr:Bootstrap complete. 2024-04-02T14:56:40.552 INFO:tasks.cephadm:Fetching config... 2024-04-02T14:56:40.552 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-04-02T14:56:40.552 DEBUG:teuthology.orchestra.run.smithi042:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-02T14:56:40.570 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-02T14:56:40.571 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-04-02T14:56:40.571 DEBUG:teuthology.orchestra.run.smithi042:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-02T14:56:40.625 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-02T14:56:40.625 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-04-02T14:56:40.625 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd if=/var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/keyring of=/dev/stdout 2024-04-02T14:56:40.692 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-02T14:56:40.692 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-04-02T14:56:40.693 DEBUG:teuthology.orchestra.run.smithi042:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-02T14:56:40.747 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-02T14:56:40.747 DEBUG:teuthology.orchestra.run.smithi042:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD5Dvxf/nkW3/W3wJatdfdBD+WE3xgdsCTDIyX2eUuUsaPIpk/kHPyB9O1+lNCK1eHLxkgDOMxPacbedm6MXuru/XAe9VNz0GQ6D9Ev0Lf9DSxTkfoGKdwLVdRbxrgYNVbX/5Zxo6wrGvxSEmYXBb4G2VHpdT7vIk/F+RJNaripCD8jMiOKXvCUis1iSLWtaWyU09GMTFirvFmIE6zgzLXpCpkPhFWP69IvVWxxD9liSq2YHrNe2q3LX3+TbW3cNxJZib99wSeaS1Z1XN+pkAzrNxnkKEKdwk4LCjlVYYjXH5JasXXUW/59/bT3NphUw8mqc472di+/4PjPO/0Eso8s7KhlMyqqwhJlaY2fSY8VbsMPxsTWCBVAwayf8D0oAYFHy7KxPc0N+glTAT7Gv13y2T/ogOwxsAzuXPSFaUdyxdh+PYH4NtJz9YjI7IQZHpp3l+3i3SjVf/thXvynLjA6TqPxFcVHgq5Pe/KD1mKJv0ElAmV1kVQY7PGM+sVZQHU= ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-02T14:56:40.827 INFO:teuthology.orchestra.run.smithi042.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD5Dvxf/nkW3/W3wJatdfdBD+WE3xgdsCTDIyX2eUuUsaPIpk/kHPyB9O1+lNCK1eHLxkgDOMxPacbedm6MXuru/XAe9VNz0GQ6D9Ev0Lf9DSxTkfoGKdwLVdRbxrgYNVbX/5Zxo6wrGvxSEmYXBb4G2VHpdT7vIk/F+RJNaripCD8jMiOKXvCUis1iSLWtaWyU09GMTFirvFmIE6zgzLXpCpkPhFWP69IvVWxxD9liSq2YHrNe2q3LX3+TbW3cNxJZib99wSeaS1Z1XN+pkAzrNxnkKEKdwk4LCjlVYYjXH5JasXXUW/59/bT3NphUw8mqc472di+/4PjPO/0Eso8s7KhlMyqqwhJlaY2fSY8VbsMPxsTWCBVAwayf8D0oAYFHy7KxPc0N+glTAT7Gv13y2T/ogOwxsAzuXPSFaUdyxdh+PYH4NtJz9YjI7IQZHpp3l+3i3SjVf/thXvynLjA6TqPxFcVHgq5Pe/KD1mKJv0ElAmV1kVQY7PGM+sVZQHU= ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:56:40.841 DEBUG:teuthology.orchestra.run.smithi169:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD5Dvxf/nkW3/W3wJatdfdBD+WE3xgdsCTDIyX2eUuUsaPIpk/kHPyB9O1+lNCK1eHLxkgDOMxPacbedm6MXuru/XAe9VNz0GQ6D9Ev0Lf9DSxTkfoGKdwLVdRbxrgYNVbX/5Zxo6wrGvxSEmYXBb4G2VHpdT7vIk/F+RJNaripCD8jMiOKXvCUis1iSLWtaWyU09GMTFirvFmIE6zgzLXpCpkPhFWP69IvVWxxD9liSq2YHrNe2q3LX3+TbW3cNxJZib99wSeaS1Z1XN+pkAzrNxnkKEKdwk4LCjlVYYjXH5JasXXUW/59/bT3NphUw8mqc472di+/4PjPO/0Eso8s7KhlMyqqwhJlaY2fSY8VbsMPxsTWCBVAwayf8D0oAYFHy7KxPc0N+glTAT7Gv13y2T/ogOwxsAzuXPSFaUdyxdh+PYH4NtJz9YjI7IQZHpp3l+3i3SjVf/thXvynLjA6TqPxFcVHgq5Pe/KD1mKJv0ElAmV1kVQY7PGM+sVZQHU= ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-02T14:56:40.890 INFO:teuthology.orchestra.run.smithi169.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD5Dvxf/nkW3/W3wJatdfdBD+WE3xgdsCTDIyX2eUuUsaPIpk/kHPyB9O1+lNCK1eHLxkgDOMxPacbedm6MXuru/XAe9VNz0GQ6D9Ev0Lf9DSxTkfoGKdwLVdRbxrgYNVbX/5Zxo6wrGvxSEmYXBb4G2VHpdT7vIk/F+RJNaripCD8jMiOKXvCUis1iSLWtaWyU09GMTFirvFmIE6zgzLXpCpkPhFWP69IvVWxxD9liSq2YHrNe2q3LX3+TbW3cNxJZib99wSeaS1Z1XN+pkAzrNxnkKEKdwk4LCjlVYYjXH5JasXXUW/59/bT3NphUw8mqc472di+/4PjPO/0Eso8s7KhlMyqqwhJlaY2fSY8VbsMPxsTWCBVAwayf8D0oAYFHy7KxPc0N+glTAT7Gv13y2T/ogOwxsAzuXPSFaUdyxdh+PYH4NtJz9YjI7IQZHpp3l+3i3SjVf/thXvynLjA6TqPxFcVHgq5Pe/KD1mKJv0ElAmV1kVQY7PGM+sVZQHU= ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:56:40.905 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-02T14:56:41.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:40 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/2325251379' entity='client.admin' 2024-04-02T14:56:44.034 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-02T14:56:44.034 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-02T14:56:44.595 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:44 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3040424197' entity='client.admin' 2024-04-02T14:56:46.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:46 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:46.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:46 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi042", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:56:46.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:46 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:46.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:46 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:46.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:46 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:46.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:46 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:47.115 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi169 2024-04-02T14:56:47.115 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-04-02T14:56:47.115 DEBUG:teuthology.orchestra.run.smithi169:> dd of=/etc/ceph/ceph.conf 2024-04-02T14:56:47.133 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-04-02T14:56:47.133 DEBUG:teuthology.orchestra.run.smithi169:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-02T14:56:47.190 INFO:tasks.cephadm:Adding host smithi169 to orchestrator... 2024-04-02T14:56:47.191 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch host add smithi169 2024-04-02T14:56:47.704 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:47 smithi042 ceph-mon[24805]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:47.704 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:47 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:47.705 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:47 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:56:47.705 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:47 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:56:47.705 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:47 smithi042 ceph-mon[24805]: Updating smithi042:/etc/ceph/ceph.conf 2024-04-02T14:56:47.705 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:47 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:47.705 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:47 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:48.514 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:48 smithi042 ceph-mon[24805]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-04-02T14:56:49.981 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:49 smithi042 ceph-mon[24805]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi169", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:49.982 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:49 smithi042 ceph-mon[24805]: Deploying cephadm binary to smithi169 2024-04-02T14:56:49.982 INFO:teuthology.orchestra.run.smithi042.stdout:Added host 'smithi169' with addr '172.21.15.169' 2024-04-02T14:56:50.530 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch host ls --format=json 2024-04-02T14:56:51.108 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:50 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:51.108 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:50 smithi042 ceph-mon[24805]: Added host smithi169 2024-04-02T14:56:51.108 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:50 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:51.108 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:50 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:56:51.108 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:50 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:56:51.108 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:50 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:51.108 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:50 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:52.304 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T14:56:52.305 INFO:teuthology.orchestra.run.smithi042.stdout:[{"addr": "172.21.15.42", "hostname": "smithi042", "labels": [], "status": ""}, {"addr": "172.21.15.169", "hostname": "smithi169", "labels": [], "status": ""}] 2024-04-02T14:56:52.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:52 smithi042 ceph-mon[24805]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:56:52.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:52 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:52.892 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-02T14:56:52.893 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph osd crush tunables default 2024-04-02T14:56:53.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:53 smithi042 ceph-mon[24805]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T14:56:54.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:54 smithi042 ceph-mon[24805]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:56:54.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:54 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:54.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:54 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:54.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:54 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi169", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:56:54.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:54 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:55.590 INFO:teuthology.orchestra.run.smithi042.stderr:adjusted tunables profile to default 2024-04-02T14:56:55.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:55 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1337602533' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-02T14:56:56.134 INFO:tasks.cephadm:Adding mon.a on smithi042 2024-04-02T14:56:56.134 INFO:tasks.cephadm:Adding mon.c on smithi042 2024-04-02T14:56:56.135 INFO:tasks.cephadm:Adding mon.b on smithi169 2024-04-02T14:56:56.135 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch apply mon '3;smithi042:172.21.15.42=a;smithi042:[v2:172.21.15.42:3301,v1:172.21.15.42:6790]=c;smithi169:172.21.15.169=b' 2024-04-02T14:56:57.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:56 smithi042 ceph-mon[24805]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:56:57.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:56 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1337602533' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-02T14:56:57.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:56 smithi042 ceph-mon[24805]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T14:56:57.955 INFO:teuthology.orchestra.run.smithi169.stdout:Scheduled mon update... 2024-04-02T14:56:58.606 DEBUG:teuthology.orchestra.run.smithi042:mon.c> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mon.c.service 2024-04-02T14:56:58.609 DEBUG:teuthology.orchestra.run.smithi169:mon.b> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mon.b.service 2024-04-02T14:56:58.611 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-02T14:56:58.611 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:56:58.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:58 smithi042 ceph-mon[24805]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:56:58.892 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:58 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:58.892 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:58 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:58.892 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:58 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:56:58.892 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:58 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:56:58.892 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:58 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:58.892 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:58 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:56:58.892 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:58 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:56:58.892 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:58 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:56:59.895 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:59 smithi042 ceph-mon[24805]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi042:172.21.15.42=a;smithi042:[v2:172.21.15.42:3301,v1:172.21.15.42:6790]=c;smithi169:172.21.15.169=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:59.895 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:59 smithi042 ceph-mon[24805]: Saving service mon spec with placement smithi042:172.21.15.42=a;smithi042:[v2:172.21.15.42:3301,v1:172.21.15.42:6790]=c;smithi169:172.21.15.169=b;count:3 2024-04-02T14:56:59.895 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:59 smithi042 ceph-mon[24805]: Updating smithi169:/etc/ceph/ceph.conf 2024-04-02T14:56:59.896 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:59 smithi042 ceph-mon[24805]: Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2024-04-02T14:56:59.896 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:56:59 smithi042 ceph-mon[24805]: Deploying daemon mon.c on smithi042 2024-04-02T14:57:00.528 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-04-02T14:57:00.528 INFO:teuthology.orchestra.run.smithi169.stdout:{"epoch":1,"fsid":"fd27f2e2-f100-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:55:38.108567Z","created":"2024-04-02T14:55:38.108567Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T14:57:00.529 INFO:teuthology.orchestra.run.smithi169.stderr:dumped monmap epoch 1 2024-04-02T14:57:00.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:00 smithi042 ceph-mon[24805]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:00.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:00 smithi042 ceph-mon[24805]: from='client.? 172.21.15.169:0/586803967' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:57:02.115 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-02T14:57:02.116 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:57:06.997 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-04-02T14:57:06.998 INFO:teuthology.orchestra.run.smithi169.stdout:{"epoch":2,"fsid":"fd27f2e2-f100-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:57:01.688098Z","created":"2024-04-02T14:55:38.108567Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3301","nonce":0},{"type":"v1","addr":"172.21.15.42:6790","nonce":0}]},"addr":"172.21.15.42:6790/0","public_addr":"172.21.15.42:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-02T14:57:07.000 INFO:teuthology.orchestra.run.smithi169.stderr:dumped monmap epoch 2 2024-04-02T14:57:07.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:07.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:57:07.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: mon.a calling monitor election 2024-04-02T14:57:07.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:07.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:07.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:07.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:07.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: mon.c calling monitor election 2024-04-02T14:57:07.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:07.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:07.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:07.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:07.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-02T14:57:07.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: monmap e2: 2 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],c=[v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0]} 2024-04-02T14:57:07.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: fsmap 2024-04-02T14:57:07.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T14:57:07.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: mgrmap e13: y(active, since 36s) 2024-04-02T14:57:07.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: overall HEALTH_OK 2024-04-02T14:57:07.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:07.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:57:07.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:06 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:08.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:07 smithi042 ceph-mon[24805]: Deploying daemon mon.b on smithi169 2024-04-02T14:57:08.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:07 smithi042 ceph-mon[24805]: from='client.? 172.21.15.169:0/2137035938' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:57:08.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:07 smithi042 ceph-mon[24805]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:08.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:07 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:08.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:07 smithi042 ceph-mon[29473]: Deploying daemon mon.b on smithi169 2024-04-02T14:57:08.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:07 smithi042 ceph-mon[29473]: from='client.? 172.21.15.169:0/2137035938' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:57:08.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:07 smithi042 ceph-mon[29473]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:08.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:07 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:08.580 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-02T14:57:08.581 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:57:10.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:10 smithi042 ceph-mon[24805]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:10.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:10 smithi042 ceph-mon[29473]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:11.335 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-04-02T14:57:11.335 INFO:teuthology.orchestra.run.smithi169.stdout:{"epoch":2,"fsid":"fd27f2e2-f100-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:57:01.688098Z","created":"2024-04-02T14:55:38.108567Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3301","nonce":0},{"type":"v1","addr":"172.21.15.42:6790","nonce":0}]},"addr":"172.21.15.42:6790/0","public_addr":"172.21.15.42:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-02T14:57:11.336 INFO:teuthology.orchestra.run.smithi169.stderr:dumped monmap epoch 2 2024-04-02T14:57:11.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.sst_partitioner_factory: None 2024-04-02T14:57:11.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-02T14:57:11.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-02T14:57:11.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55597adc5d00) 2024-04-02T14:57:11.439 INFO:journalctl@ceph.mon.b.smithi169.stdout: cache_index_and_filter_blocks: 1 2024-04-02T14:57:11.440 INFO:journalctl@ceph.mon.b.smithi169.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-02T14:57:11.440 INFO:journalctl@ceph.mon.b.smithi169.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-02T14:57:11.440 INFO:journalctl@ceph.mon.b.smithi169.stdout: pin_top_level_index_and_filter: 1 2024-04-02T14:57:11.440 INFO:journalctl@ceph.mon.b.smithi169.stdout: index_type: 0 2024-04-02T14:57:11.440 INFO:journalctl@ceph.mon.b.smithi169.stdout: data_block_index_type: 0 2024-04-02T14:57:11.440 INFO:journalctl@ceph.mon.b.smithi169.stdout: index_shortening: 1 2024-04-02T14:57:11.440 INFO:journalctl@ceph.mon.b.smithi169.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-02T14:57:11.440 INFO:journalctl@ceph.mon.b.smithi169.stdout: hash_index_allow_collision: 1 2024-04-02T14:57:11.440 INFO:journalctl@ceph.mon.b.smithi169.stdout: checksum: 1 2024-04-02T14:57:11.440 INFO:journalctl@ceph.mon.b.smithi169.stdout: no_block_cache: 0 2024-04-02T14:57:11.440 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_cache: 0x55597ae30170 2024-04-02T14:57:11.440 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_cache_name: BinnedLRUCache 2024-04-02T14:57:11.440 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_cache_options: 2024-04-02T14:57:11.441 INFO:journalctl@ceph.mon.b.smithi169.stdout: capacity : 536870912 2024-04-02T14:57:11.441 INFO:journalctl@ceph.mon.b.smithi169.stdout: num_shard_bits : 4 2024-04-02T14:57:11.441 INFO:journalctl@ceph.mon.b.smithi169.stdout: strict_capacity_limit : 0 2024-04-02T14:57:11.441 INFO:journalctl@ceph.mon.b.smithi169.stdout: high_pri_pool_ratio: 0.000 2024-04-02T14:57:11.441 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_cache_compressed: (nil) 2024-04-02T14:57:11.441 INFO:journalctl@ceph.mon.b.smithi169.stdout: persistent_cache: (nil) 2024-04-02T14:57:11.441 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_size: 4096 2024-04-02T14:57:11.441 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_size_deviation: 10 2024-04-02T14:57:11.441 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_restart_interval: 16 2024-04-02T14:57:11.441 INFO:journalctl@ceph.mon.b.smithi169.stdout: index_block_restart_interval: 1 2024-04-02T14:57:11.441 INFO:journalctl@ceph.mon.b.smithi169.stdout: metadata_block_size: 4096 2024-04-02T14:57:11.441 INFO:journalctl@ceph.mon.b.smithi169.stdout: partition_filters: 0 2024-04-02T14:57:11.441 INFO:journalctl@ceph.mon.b.smithi169.stdout: use_delta_encoding: 1 2024-04-02T14:57:11.441 INFO:journalctl@ceph.mon.b.smithi169.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-02T14:57:11.442 INFO:journalctl@ceph.mon.b.smithi169.stdout: whole_key_filtering: 1 2024-04-02T14:57:11.442 INFO:journalctl@ceph.mon.b.smithi169.stdout: verify_compression: 0 2024-04-02T14:57:11.442 INFO:journalctl@ceph.mon.b.smithi169.stdout: read_amp_bytes_per_bit: 0 2024-04-02T14:57:11.442 INFO:journalctl@ceph.mon.b.smithi169.stdout: format_version: 4 2024-04-02T14:57:11.442 INFO:journalctl@ceph.mon.b.smithi169.stdout: enable_index_compression: 1 2024-04-02T14:57:11.442 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_align: 0 2024-04-02T14:57:11.442 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.write_buffer_size: 33554432 2024-04-02T14:57:11.442 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.max_write_buffer_number: 2 2024-04-02T14:57:11.442 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compression: NoCompression 2024-04-02T14:57:11.442 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.bottommost_compression: Disabled 2024-04-02T14:57:11.442 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.prefix_extractor: nullptr 2024-04-02T14:57:11.442 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-02T14:57:11.442 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.num_levels: 7 2024-04-02T14:57:11.442 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-02T14:57:11.443 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-02T14:57:11.443 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-02T14:57:11.443 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-02T14:57:11.443 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-02T14:57:11.443 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-02T14:57:11.443 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-02T14:57:11.443 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compression_opts.level: 32767 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compression_opts.strategy: 0 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compression_opts.enabled: false 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.target_file_size_base: 67108864 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-02T14:57:11.444 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.arena_block_size: 4194304 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.disable_auto_compactions: 0 2024-04-02T14:57:11.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-02T14:57:11.446 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-02T14:57:11.446 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-02T14:57:11.446 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-02T14:57:11.446 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-02T14:57:11.446 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-02T14:57:11.446 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-02T14:57:11.446 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-02T14:57:11.446 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-02T14:57:11.446 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-02T14:57:11.446 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.table_properties_collectors: 2024-04-02T14:57:11.446 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.inplace_update_support: 0 2024-04-02T14:57:11.446 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-02T14:57:11.446 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-02T14:57:11.446 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.bloom_locality: 0 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.max_successive_merges: 0 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.paranoid_file_checks: 0 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.force_consistency_checks: 1 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.report_bg_io_stats: 0 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.ttl: 2592000 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.enable_blob_files: false 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.min_blob_size: 0 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.blob_file_size: 268435456 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-02T14:57:11.447 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-02T14:57:11.448 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-02T14:57:11.448 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-02T14:57:11.448 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712069831448009, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-02T14:57:11.448 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-02T14:57:11.448 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-02T14:57:11.448 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712069831448334, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712069831, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "b832dc2f-b4cd-49a2-a8d4-e3c2b40fcc59", "db_session_id": "EHTVESVP6HYIPRM40DU0"}} 2024-04-02T14:57:11.448 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-02T14:57:11.448 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712069831448639, "job": 1, "event": "recovery_finished"} 2024-04-02T14:57:11.448 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-02T14:57:11.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55597ae16a80 2024-04-02T14:57:11.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: DB pointer 0x55597ae26000 2024-04-02T14:57:11.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-02T14:57:11.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-02T14:57:11.730 INFO:journalctl@ceph.mon.b.smithi169.stdout: ** DB Stats ** 2024-04-02T14:57:11.730 INFO:journalctl@ceph.mon.b.smithi169.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T14:57:11.730 INFO:journalctl@ceph.mon.b.smithi169.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-02T14:57:11.730 INFO:journalctl@ceph.mon.b.smithi169.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-02T14:57:11.730 INFO:journalctl@ceph.mon.b.smithi169.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T14:57:11.730 INFO:journalctl@ceph.mon.b.smithi169.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-02T14:57:11.730 INFO:journalctl@ceph.mon.b.smithi169.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-02T14:57:11.730 INFO:journalctl@ceph.mon.b.smithi169.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T14:57:11.730 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2024-04-02T14:57:11.730 INFO:journalctl@ceph.mon.b.smithi169.stdout: ** Compaction Stats [default] ** 2024-04-02T14:57:11.730 INFO:journalctl@ceph.mon.b.smithi169.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T14:57:11.730 INFO:journalctl@ceph.mon.b.smithi169.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T14:57:11.731 INFO:journalctl@ceph.mon.b.smithi169.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-02T14:57:11.731 INFO:journalctl@ceph.mon.b.smithi169.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-02T14:57:11.731 INFO:journalctl@ceph.mon.b.smithi169.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-02T14:57:11.731 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2024-04-02T14:57:11.731 INFO:journalctl@ceph.mon.b.smithi169.stdout: ** Compaction Stats [default] ** 2024-04-02T14:57:11.731 INFO:journalctl@ceph.mon.b.smithi169.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T14:57:11.731 INFO:journalctl@ceph.mon.b.smithi169.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T14:57:11.731 INFO:journalctl@ceph.mon.b.smithi169.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-02T14:57:11.731 INFO:journalctl@ceph.mon.b.smithi169.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T14:57:11.731 INFO:journalctl@ceph.mon.b.smithi169.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T14:57:11.731 INFO:journalctl@ceph.mon.b.smithi169.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T14:57:11.731 INFO:journalctl@ceph.mon.b.smithi169.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T14:57:11.731 INFO:journalctl@ceph.mon.b.smithi169.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T14:57:11.731 INFO:journalctl@ceph.mon.b.smithi169.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T14:57:11.731 INFO:journalctl@ceph.mon.b.smithi169.stdout: Cumulative compaction: 0.00 GB write, 0.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T14:57:11.732 INFO:journalctl@ceph.mon.b.smithi169.stdout: Interval compaction: 0.00 GB write, 0.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T14:57:11.732 INFO:journalctl@ceph.mon.b.smithi169.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T14:57:11.732 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2024-04-02T14:57:11.732 INFO:journalctl@ceph.mon.b.smithi169.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T14:57:11.732 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2024-04-02T14:57:11.732 INFO:journalctl@ceph.mon.b.smithi169.stdout: ** Compaction Stats [default] ** 2024-04-02T14:57:11.732 INFO:journalctl@ceph.mon.b.smithi169.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T14:57:11.732 INFO:journalctl@ceph.mon.b.smithi169.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T14:57:11.732 INFO:journalctl@ceph.mon.b.smithi169.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-02T14:57:11.732 INFO:journalctl@ceph.mon.b.smithi169.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-02T14:57:11.732 INFO:journalctl@ceph.mon.b.smithi169.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-02T14:57:11.732 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2024-04-02T14:57:11.732 INFO:journalctl@ceph.mon.b.smithi169.stdout: ** Compaction Stats [default] ** 2024-04-02T14:57:11.732 INFO:journalctl@ceph.mon.b.smithi169.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T14:57:11.733 INFO:journalctl@ceph.mon.b.smithi169.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T14:57:11.733 INFO:journalctl@ceph.mon.b.smithi169.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-02T14:57:11.733 INFO:journalctl@ceph.mon.b.smithi169.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T14:57:11.733 INFO:journalctl@ceph.mon.b.smithi169.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T14:57:11.733 INFO:journalctl@ceph.mon.b.smithi169.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T14:57:11.733 INFO:journalctl@ceph.mon.b.smithi169.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T14:57:11.733 INFO:journalctl@ceph.mon.b.smithi169.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T14:57:11.733 INFO:journalctl@ceph.mon.b.smithi169.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T14:57:11.733 INFO:journalctl@ceph.mon.b.smithi169.stdout: Cumulative compaction: 0.00 GB write, 0.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T14:57:11.733 INFO:journalctl@ceph.mon.b.smithi169.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T14:57:11.733 INFO:journalctl@ceph.mon.b.smithi169.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T14:57:11.733 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2024-04-02T14:57:11.733 INFO:journalctl@ceph.mon.b.smithi169.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T14:57:11.733 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-02T14:57:11.733 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: using public_addr v2:172.21.15.169:0/0 -> [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] 2024-04-02T14:57:11.734 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: starting mon.b rank -1 at public addrs [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] at bind addrs [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:57:11.734 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b@-1(???) e0 preinit fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:57:11.734 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b@-1(synchronizing).mds e1 new map 2024-04-02T14:57:11.734 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-02T14:57:11.734 INFO:journalctl@ceph.mon.b.smithi169.stdout: e1 2024-04-02T14:57:11.734 INFO:journalctl@ceph.mon.b.smithi169.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T14:57:11.734 INFO:journalctl@ceph.mon.b.smithi169.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T14:57:11.734 INFO:journalctl@ceph.mon.b.smithi169.stdout: legacy client fscid: -1 2024-04-02T14:57:11.734 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2024-04-02T14:57:11.734 INFO:journalctl@ceph.mon.b.smithi169.stdout: No filesystems configured 2024-04-02T14:57:11.734 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-02T14:57:11.734 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-02T14:57:11.734 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-02T14:57:11.734 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-02T14:57:11.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-02T14:57:11.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-02T14:57:11.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-02T14:57:11.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T14:57:11.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T14:57:11.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T14:57:11.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:11.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: from='client.? 172.21.15.169:0/586803967' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:57:11.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:57:11.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.a calling monitor election 2024-04-02T14:57:11.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:11.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:11.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:11.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:11.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.c calling monitor election 2024-04-02T14:57:11.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:11.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:11.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:11.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:11.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-02T14:57:11.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: monmap e2: 2 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],c=[v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0]} 2024-04-02T14:57:11.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: fsmap 2024-04-02T14:57:11.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T14:57:11.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mgrmap e13: y(active, since 36s) 2024-04-02T14:57:11.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: overall HEALTH_OK 2024-04-02T14:57:11.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:11.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:57:11.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:11.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: Deploying daemon mon.b on smithi169 2024-04-02T14:57:11.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: from='client.? 172.21.15.169:0/2137035938' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:57:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-02T14:57:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 ceph-mon[25801]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-02T14:57:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 bash[25767]: 1cc495c7e481ad4e309d21d89c2dc8a88cd3a7b202fe7e8f407a515ed9112e37 2024-04-02T14:57:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:11 smithi169 systemd[1]: Started Ceph mon.b for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T14:57:12.950 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-02T14:57:12.951 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:57:16.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:16.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:57:16.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: mon.a calling monitor election 2024-04-02T14:57:16.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: mon.c calling monitor election 2024-04-02T14:57:16.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:16.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:16.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:16.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:16.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:16.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:16.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:16.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:16.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:16.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-02T14:57:16.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: monmap e3: 3 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0]} 2024-04-02T14:57:16.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: fsmap 2024-04-02T14:57:16.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T14:57:16.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: mgrmap e13: y(active, since 45s) 2024-04-02T14:57:16.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-02T14:57:16.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-02T14:57:16.797 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-02T14:57:16.797 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: mon.b (rank 2) addr [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] is down (out of quorum) 2024-04-02T14:57:16.797 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:16.797 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:16.797 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:16.797 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:16.797 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:16.797 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:57:16.797 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: mon.a calling monitor election 2024-04-02T14:57:16.797 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: mon.c calling monitor election 2024-04-02T14:57:16.797 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:16.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:16.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:16.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:16.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:16.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:16.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:16.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:16.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:16.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-02T14:57:16.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: monmap e3: 3 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0]} 2024-04-02T14:57:16.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: fsmap 2024-04-02T14:57:16.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T14:57:16.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: mgrmap e13: y(active, since 45s) 2024-04-02T14:57:16.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-02T14:57:16.799 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-02T14:57:16.799 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-02T14:57:16.799 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: mon.b (rank 2) addr [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] is down (out of quorum) 2024-04-02T14:57:16.799 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:16.799 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:16.799 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:16.799 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:16 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:17.831 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-04-02T14:57:17.831 INFO:teuthology.orchestra.run.smithi169.stdout:{"epoch":3,"fsid":"fd27f2e2-f100-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:57:11.470865Z","created":"2024-04-02T14:55:38.108567Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3301","nonce":0},{"type":"v1","addr":"172.21.15.42:6790","nonce":0}]},"addr":"172.21.15.42:6790/0","public_addr":"172.21.15.42:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:3300","nonce":0},{"type":"v1","addr":"172.21.15.169:6789","nonce":0}]},"addr":"172.21.15.169:6789/0","public_addr":"172.21.15.169:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-02T14:57:17.833 INFO:teuthology.orchestra.run.smithi169.stderr:dumped monmap epoch 3 2024-04-02T14:57:18.373 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-02T14:57:18.373 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph config generate-minimal-conf 2024-04-02T14:57:18.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[24805]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:18.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[24805]: Updating smithi169:/etc/ceph/ceph.conf 2024-04-02T14:57:18.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:18.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[24805]: from='client.? 172.21.15.169:0/1642453416' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:57:18.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:57:18.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T14:57:18.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:18.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[29473]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:18.546 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.546 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[29473]: Updating smithi169:/etc/ceph/ceph.conf 2024-04-02T14:57:18.546 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:18.546 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.546 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.546 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[29473]: from='client.? 172.21.15.169:0/1642453416' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:57:18.546 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.546 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.546 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:57:18.546 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T14:57:18.546 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:18.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:18.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:57:18.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: mon.a calling monitor election 2024-04-02T14:57:18.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: mon.c calling monitor election 2024-04-02T14:57:18.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:18.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:57:18.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:18.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:18.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:18.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:18.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:18.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:18.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:18.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-02T14:57:18.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: monmap e3: 3 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0]} 2024-04-02T14:57:18.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: fsmap 2024-04-02T14:57:18.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T14:57:18.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: mgrmap e13: y(active, since 45s) 2024-04-02T14:57:18.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-02T14:57:18.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-02T14:57:18.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-02T14:57:18.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: mon.b (rank 2) addr [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] is down (out of quorum) 2024-04-02T14:57:18.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:18.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:18.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:18.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:18.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: Updating smithi169:/etc/ceph/ceph.conf 2024-04-02T14:57:18.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:18.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='client.? 172.21.15.169:0/1642453416' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:57:18.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:18.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:57:18.732 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T14:57:18.732 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:19.535 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[24805]: mon.b calling monitor election 2024-04-02T14:57:19.535 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[24805]: Updating smithi042:/etc/ceph/ceph.conf 2024-04-02T14:57:19.535 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[24805]: Reconfiguring mon.a (unknown last config time)... 2024-04-02T14:57:19.535 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[24805]: Reconfiguring daemon mon.a on smithi042 2024-04-02T14:57:19.535 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[24805]: mon.b calling monitor election 2024-04-02T14:57:19.536 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[24805]: mon.c calling monitor election 2024-04-02T14:57:19.536 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[24805]: mon.a calling monitor election 2024-04-02T14:57:19.536 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[24805]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T14:57:19.536 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[24805]: monmap e3: 3 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0]} 2024-04-02T14:57:19.536 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[24805]: fsmap 2024-04-02T14:57:19.536 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[24805]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T14:57:19.536 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[24805]: mgrmap e13: y(active, since 47s) 2024-04-02T14:57:19.536 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[24805]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-02T14:57:19.536 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[24805]: Cluster is now healthy 2024-04-02T14:57:19.536 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[24805]: overall HEALTH_OK 2024-04-02T14:57:19.536 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[29473]: mon.b calling monitor election 2024-04-02T14:57:19.536 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[29473]: Updating smithi042:/etc/ceph/ceph.conf 2024-04-02T14:57:19.536 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[29473]: Reconfiguring mon.a (unknown last config time)... 2024-04-02T14:57:19.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[29473]: Reconfiguring daemon mon.a on smithi042 2024-04-02T14:57:19.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[29473]: mon.b calling monitor election 2024-04-02T14:57:19.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[29473]: mon.c calling monitor election 2024-04-02T14:57:19.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[29473]: mon.a calling monitor election 2024-04-02T14:57:19.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[29473]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T14:57:19.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[29473]: monmap e3: 3 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0]} 2024-04-02T14:57:19.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[29473]: fsmap 2024-04-02T14:57:19.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[29473]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T14:57:19.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[29473]: mgrmap e13: y(active, since 47s) 2024-04-02T14:57:19.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[29473]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-02T14:57:19.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[29473]: Cluster is now healthy 2024-04-02T14:57:19.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:19 smithi042 ceph-mon[29473]: overall HEALTH_OK 2024-04-02T14:57:19.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:19 smithi169 ceph-mon[25801]: mon.b calling monitor election 2024-04-02T14:57:19.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:19 smithi169 ceph-mon[25801]: Updating smithi042:/etc/ceph/ceph.conf 2024-04-02T14:57:19.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:19 smithi169 ceph-mon[25801]: Reconfiguring mon.a (unknown last config time)... 2024-04-02T14:57:19.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:19 smithi169 ceph-mon[25801]: Reconfiguring daemon mon.a on smithi042 2024-04-02T14:57:19.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:19 smithi169 ceph-mon[25801]: mon.b calling monitor election 2024-04-02T14:57:19.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:19 smithi169 ceph-mon[25801]: mon.c calling monitor election 2024-04-02T14:57:19.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:19 smithi169 ceph-mon[25801]: mon.a calling monitor election 2024-04-02T14:57:19.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:19 smithi169 ceph-mon[25801]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T14:57:19.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:19 smithi169 ceph-mon[25801]: monmap e3: 3 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0]} 2024-04-02T14:57:19.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:19 smithi169 ceph-mon[25801]: fsmap 2024-04-02T14:57:19.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:19 smithi169 ceph-mon[25801]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T14:57:19.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:19 smithi169 ceph-mon[25801]: mgrmap e13: y(active, since 47s) 2024-04-02T14:57:19.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:19 smithi169 ceph-mon[25801]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-02T14:57:19.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:19 smithi169 ceph-mon[25801]: Cluster is now healthy 2024-04-02T14:57:19.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:19 smithi169 ceph-mon[25801]: overall HEALTH_OK 2024-04-02T14:57:20.395 INFO:teuthology.orchestra.run.smithi042.stdout:# minimal ceph.conf for fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:57:20.395 INFO:teuthology.orchestra.run.smithi042.stdout:[global] 2024-04-02T14:57:20.395 INFO:teuthology.orchestra.run.smithi042.stdout: fsid = fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T14:57:20.395 INFO:teuthology.orchestra.run.smithi042.stdout: mon_host = [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] 2024-04-02T14:57:20.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:20 smithi042 ceph-mon[29473]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:20.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:20 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:20.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:20 smithi042 ceph-mon[24805]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:20.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:20 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:20.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:20 smithi169 ceph-mon[25801]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:20.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:20 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:57:21.336 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-02T14:57:21.336 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-04-02T14:57:21.336 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/etc/ceph/ceph.conf 2024-04-02T14:57:21.365 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-04-02T14:57:21.365 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-02T14:57:21.432 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-04-02T14:57:21.432 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/ceph/ceph.conf 2024-04-02T14:57:21.465 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-04-02T14:57:21.465 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-02T14:57:21.536 INFO:tasks.cephadm:Adding mgr.y on smithi042 2024-04-02T14:57:21.536 INFO:tasks.cephadm:Adding mgr.x on smithi169 2024-04-02T14:57:21.536 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch apply mgr '2;smithi042=y;smithi169=x' 2024-04-02T14:57:21.597 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:21 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/979071033' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:21.597 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:21 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:21.597 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:21 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:57:21.597 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:21 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T14:57:21.597 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:21 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:21.597 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:21 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:21.597 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:21 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/979071033' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:21.597 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:21 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:21.597 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:21 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:57:21.597 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:21 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T14:57:21.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:21 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:21.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:21 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:21.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:21 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/979071033' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:21.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:21 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:21.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:21 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:57:21.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:21 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T14:57:21.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:21 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:21.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:21 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:22.431 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:22 smithi169 ceph-mon[25801]: Reconfiguring mon.c (monmap changed)... 2024-04-02T14:57:22.431 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:22 smithi169 ceph-mon[25801]: Reconfiguring daemon mon.c on smithi042 2024-04-02T14:57:22.431 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:22 smithi169 ceph-mon[25801]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:22.432 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:22 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:22.432 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:22 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:57:22.432 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:22 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T14:57:22.432 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:22 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:22.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:22 smithi042 ceph-mon[24805]: Reconfiguring mon.c (monmap changed)... 2024-04-02T14:57:22.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:22 smithi042 ceph-mon[24805]: Reconfiguring daemon mon.c on smithi042 2024-04-02T14:57:22.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:22 smithi042 ceph-mon[24805]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:22.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:22 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:22.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:22 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:57:22.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:22 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T14:57:22.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:22 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:22.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:22 smithi042 ceph-mon[29473]: Reconfiguring mon.c (monmap changed)... 2024-04-02T14:57:22.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:22 smithi042 ceph-mon[29473]: Reconfiguring daemon mon.c on smithi042 2024-04-02T14:57:22.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:22 smithi042 ceph-mon[29473]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:22.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:22 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:22.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:22 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:57:22.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:22 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T14:57:22.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:22 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:23.683 INFO:teuthology.orchestra.run.smithi169.stdout:Scheduled mgr update... 2024-04-02T14:57:23.684 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:23 smithi169 ceph-mon[25801]: Reconfiguring mon.b (monmap changed)... 2024-04-02T14:57:23.684 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:23 smithi169 ceph-mon[25801]: Reconfiguring daemon mon.b on smithi169 2024-04-02T14:57:23.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:23 smithi042 ceph-mon[24805]: Reconfiguring mon.b (monmap changed)... 2024-04-02T14:57:23.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:23 smithi042 ceph-mon[24805]: Reconfiguring daemon mon.b on smithi169 2024-04-02T14:57:23.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:23 smithi042 ceph-mon[29473]: Reconfiguring mon.b (monmap changed)... 2024-04-02T14:57:23.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:23 smithi042 ceph-mon[29473]: Reconfiguring daemon mon.b on smithi169 2024-04-02T14:57:24.423 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:24 smithi169 ceph-mon[25801]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:24.423 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:24 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:24.424 DEBUG:teuthology.orchestra.run.smithi169:mgr.x> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mgr.x.service 2024-04-02T14:57:24.426 INFO:tasks.cephadm:Deploying OSDs... 2024-04-02T14:57:24.427 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-04-02T14:57:24.427 DEBUG:teuthology.orchestra.run.smithi042:> dd if=/scratch_devs of=/dev/stdout 2024-04-02T14:57:24.445 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-02T14:57:24.445 DEBUG:teuthology.orchestra.run.smithi042:> stat /dev/vg_nvme/lv_1 2024-04-02T14:57:24.503 INFO:teuthology.orchestra.run.smithi042.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-02T14:57:24.503 INFO:teuthology.orchestra.run.smithi042.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:57:24.503 INFO:teuthology.orchestra.run.smithi042.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-04-02T14:57:24.503 INFO:teuthology.orchestra.run.smithi042.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:57:24.503 INFO:teuthology.orchestra.run.smithi042.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:57:24.503 INFO:teuthology.orchestra.run.smithi042.stdout:Access: 2024-04-02 14:57:01.169367299 +0000 2024-04-02T14:57:24.503 INFO:teuthology.orchestra.run.smithi042.stdout:Modify: 2024-04-02 14:54:41.940637462 +0000 2024-04-02T14:57:24.503 INFO:teuthology.orchestra.run.smithi042.stdout:Change: 2024-04-02 14:54:41.941637454 +0000 2024-04-02T14:57:24.503 INFO:teuthology.orchestra.run.smithi042.stdout: Birth: 2024-04-02 14:54:41.940637462 +0000 2024-04-02T14:57:24.503 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-02T14:57:24.572 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records in 2024-04-02T14:57:24.572 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records out 2024-04-02T14:57:24.572 INFO:teuthology.orchestra.run.smithi042.stderr:512 bytes copied, 0.000118929 s, 4.3 MB/s 2024-04-02T14:57:24.574 DEBUG:teuthology.orchestra.run.smithi042:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-02T14:57:24.631 DEBUG:teuthology.orchestra.run.smithi042:> stat /dev/vg_nvme/lv_2 2024-04-02T14:57:24.688 INFO:teuthology.orchestra.run.smithi042.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-02T14:57:24.688 INFO:teuthology.orchestra.run.smithi042.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:57:24.688 INFO:teuthology.orchestra.run.smithi042.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-02T14:57:24.689 INFO:teuthology.orchestra.run.smithi042.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:57:24.689 INFO:teuthology.orchestra.run.smithi042.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:57:24.689 INFO:teuthology.orchestra.run.smithi042.stdout:Access: 2024-04-02 14:57:01.169367299 +0000 2024-04-02T14:57:24.689 INFO:teuthology.orchestra.run.smithi042.stdout:Modify: 2024-04-02 14:54:42.283634903 +0000 2024-04-02T14:57:24.689 INFO:teuthology.orchestra.run.smithi042.stdout:Change: 2024-04-02 14:54:42.283634903 +0000 2024-04-02T14:57:24.689 INFO:teuthology.orchestra.run.smithi042.stdout: Birth: 2024-04-02 14:54:42.283634903 +0000 2024-04-02T14:57:24.689 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-02T14:57:24.754 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:24 smithi042 ceph-mon[24805]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:24.755 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:24 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:24.755 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:24 smithi042 ceph-mon[29473]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:24.755 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:24 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:24.757 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records in 2024-04-02T14:57:24.757 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records out 2024-04-02T14:57:24.757 INFO:teuthology.orchestra.run.smithi042.stderr:512 bytes copied, 9.9716e-05 s, 5.1 MB/s 2024-04-02T14:57:24.758 DEBUG:teuthology.orchestra.run.smithi042:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-02T14:57:24.815 DEBUG:teuthology.orchestra.run.smithi042:> stat /dev/vg_nvme/lv_3 2024-04-02T14:57:24.871 INFO:teuthology.orchestra.run.smithi042.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-02T14:57:24.872 INFO:teuthology.orchestra.run.smithi042.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:57:24.872 INFO:teuthology.orchestra.run.smithi042.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-04-02T14:57:24.872 INFO:teuthology.orchestra.run.smithi042.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:57:24.872 INFO:teuthology.orchestra.run.smithi042.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:57:24.872 INFO:teuthology.orchestra.run.smithi042.stdout:Access: 2024-04-02 14:57:01.170367292 +0000 2024-04-02T14:57:24.872 INFO:teuthology.orchestra.run.smithi042.stdout:Modify: 2024-04-02 14:54:42.622632376 +0000 2024-04-02T14:57:24.872 INFO:teuthology.orchestra.run.smithi042.stdout:Change: 2024-04-02 14:54:42.622632376 +0000 2024-04-02T14:57:24.872 INFO:teuthology.orchestra.run.smithi042.stdout: Birth: 2024-04-02 14:54:42.622632376 +0000 2024-04-02T14:57:24.873 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-02T14:57:24.938 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records in 2024-04-02T14:57:24.938 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records out 2024-04-02T14:57:24.938 INFO:teuthology.orchestra.run.smithi042.stderr:512 bytes copied, 0.000107144 s, 4.8 MB/s 2024-04-02T14:57:24.939 DEBUG:teuthology.orchestra.run.smithi042:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-02T14:57:24.994 DEBUG:teuthology.orchestra.run.smithi042:> stat /dev/vg_nvme/lv_4 2024-04-02T14:57:25.050 INFO:teuthology.orchestra.run.smithi042.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-02T14:57:25.050 INFO:teuthology.orchestra.run.smithi042.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:57:25.050 INFO:teuthology.orchestra.run.smithi042.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-04-02T14:57:25.050 INFO:teuthology.orchestra.run.smithi042.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:57:25.050 INFO:teuthology.orchestra.run.smithi042.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:57:25.050 INFO:teuthology.orchestra.run.smithi042.stdout:Access: 2024-04-02 14:57:01.170367292 +0000 2024-04-02T14:57:25.050 INFO:teuthology.orchestra.run.smithi042.stdout:Modify: 2024-04-02 14:54:42.974629751 +0000 2024-04-02T14:57:25.050 INFO:teuthology.orchestra.run.smithi042.stdout:Change: 2024-04-02 14:54:42.974629751 +0000 2024-04-02T14:57:25.050 INFO:teuthology.orchestra.run.smithi042.stdout: Birth: 2024-04-02 14:54:42.974629751 +0000 2024-04-02T14:57:25.051 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-02T14:57:25.119 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records in 2024-04-02T14:57:25.119 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records out 2024-04-02T14:57:25.119 INFO:teuthology.orchestra.run.smithi042.stderr:512 bytes copied, 0.000107988 s, 4.7 MB/s 2024-04-02T14:57:25.120 DEBUG:teuthology.orchestra.run.smithi042:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-02T14:57:25.176 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-04-02T14:57:25.176 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/scratch_devs of=/dev/stdout 2024-04-02T14:57:25.191 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-02T14:57:25.191 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_1 2024-04-02T14:57:25.247 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-02T14:57:25.247 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:57:25.247 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-02T14:57:25.247 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:57:25.247 INFO:teuthology.orchestra.run.smithi169.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:57:25.247 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2024-04-02 14:57:10.276462272 +0000 2024-04-02T14:57:25.247 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2024-04-02 14:54:30.498019353 +0000 2024-04-02T14:57:25.248 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2024-04-02 14:54:30.498019353 +0000 2024-04-02T14:57:25.248 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: 2024-04-02 14:54:30.498019353 +0000 2024-04-02T14:57:25.248 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-02T14:57:25.313 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2024-04-02T14:57:25.313 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2024-04-02T14:57:25.313 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.000189173 s, 2.7 MB/s 2024-04-02T14:57:25.314 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-02T14:57:25.370 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_2 2024-04-02T14:57:25.425 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-02T14:57:25.425 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:57:25.425 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-02T14:57:25.425 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:57:25.425 INFO:teuthology.orchestra.run.smithi169.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:57:25.426 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2024-04-02 14:57:10.276462272 +0000 2024-04-02T14:57:25.426 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2024-04-02 14:54:30.857011676 +0000 2024-04-02T14:57:25.426 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2024-04-02 14:54:30.857011676 +0000 2024-04-02T14:57:25.426 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: 2024-04-02 14:54:30.857011676 +0000 2024-04-02T14:57:25.426 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-02T14:57:25.490 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2024-04-02T14:57:25.491 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2024-04-02T14:57:25.491 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.000199237 s, 2.6 MB/s 2024-04-02T14:57:25.492 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-02T14:57:25.549 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_3 2024-04-02T14:57:25.604 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-02T14:57:25.604 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:57:25.604 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-04-02T14:57:25.604 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:57:25.604 INFO:teuthology.orchestra.run.smithi169.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:57:25.604 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2024-04-02 14:57:10.277462251 +0000 2024-04-02T14:57:25.604 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2024-04-02 14:54:31.193004491 +0000 2024-04-02T14:57:25.604 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2024-04-02 14:54:31.193004491 +0000 2024-04-02T14:57:25.604 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: 2024-04-02 14:54:31.193004491 +0000 2024-04-02T14:57:25.605 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-02T14:57:25.605 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:25 smithi169 ceph-mon[25801]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi042=y;smithi169=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:57:25.605 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:25 smithi169 ceph-mon[25801]: Saving service mgr spec with placement smithi042=y;smithi169=x;count:2 2024-04-02T14:57:25.605 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:25 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:25.605 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:25 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:25.605 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:25 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:25.606 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:25 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:25.606 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:25 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:25.606 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:25 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T14:57:25.606 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:25 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-02T14:57:25.606 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:25 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T14:57:25.606 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:25 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:25.630 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2024-04-02T14:57:25.630 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2024-04-02T14:57:25.630 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.00020982 s, 2.4 MB/s 2024-04-02T14:57:25.631 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-02T14:57:25.687 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_4 2024-04-02T14:57:25.743 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-02T14:57:25.743 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:57:25.743 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-02T14:57:25.743 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:57:25.743 INFO:teuthology.orchestra.run.smithi169.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:57:25.743 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2024-04-02 14:57:10.277462251 +0000 2024-04-02T14:57:25.743 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2024-04-02 14:54:31.697993692 +0000 2024-04-02T14:57:25.743 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2024-04-02 14:54:31.697993692 +0000 2024-04-02T14:57:25.743 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: 2024-04-02 14:54:31.697993692 +0000 2024-04-02T14:57:25.743 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-02T14:57:25.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[24805]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi042=y;smithi169=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:57:25.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[24805]: Saving service mgr spec with placement smithi042=y;smithi169=x;count:2 2024-04-02T14:57:25.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:25.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:25.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:25.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:25.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:25.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T14:57:25.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-02T14:57:25.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T14:57:25.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:25.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[29473]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi042=y;smithi169=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:57:25.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[29473]: Saving service mgr spec with placement smithi042=y;smithi169=x;count:2 2024-04-02T14:57:25.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:25.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:25.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:25.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:25.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:25.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T14:57:25.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-02T14:57:25.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T14:57:25.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:25 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:25.808 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2024-04-02T14:57:25.808 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2024-04-02T14:57:25.809 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.00019624 s, 2.6 MB/s 2024-04-02T14:57:25.810 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-02T14:57:25.865 INFO:tasks.cephadm:Deploying osd.0 on smithi042 with /dev/vg_nvme/lv_4... 2024-04-02T14:57:25.866 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-02T14:57:26.486 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:26 smithi042 ceph-mon[24805]: Deploying daemon mgr.x on smithi169 2024-04-02T14:57:26.486 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:26 smithi042 ceph-mon[24805]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:26.486 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:26 smithi042 ceph-mon[29473]: Deploying daemon mgr.x on smithi169 2024-04-02T14:57:26.486 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:26 smithi042 ceph-mon[29473]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:26.643 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:26 smithi169 ceph-mon[25801]: Deploying daemon mgr.x on smithi169 2024-04-02T14:57:26.643 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:26 smithi169 ceph-mon[25801]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:26.643 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:26 smithi169 systemd[1]: Starting Ceph mgr.x for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T14:57:26.953 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:26 smithi169 podman[26966]: 2024-04-02 14:57:26.652913833 +0000 UTC m=+0.098279990 container create 794578fdbdb93f619994fc2d7cffce882fad4f6b6e5555cdb5140388171f4ef2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, release=754, com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_CLEAN=True, distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, vendor=Red Hat, Inc.) 2024-04-02T14:57:26.954 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:26 smithi169 podman[26966]: 2024-04-02 14:57:26.572823168 +0000 UTC m=+0.018189344 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T14:57:26.954 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:26 smithi169 podman[26966]: 2024-04-02 14:57:26.872511987 +0000 UTC m=+0.317878159 container init 794578fdbdb93f619994fc2d7cffce882fad4f6b6e5555cdb5140388171f4ef2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, GIT_CLEAN=True, version=8, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, release=754, com.redhat.component=centos-stream-container) 2024-04-02T14:57:26.954 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:26 smithi169 podman[26966]: 2024-04-02 14:57:26.875234885 +0000 UTC m=+0.320601050 container start 794578fdbdb93f619994fc2d7cffce882fad4f6b6e5555cdb5140388171f4ef2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, name=centos-stream, version=8, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.buildah.version=1.19.8, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , release=754, io.openshift.expose-services=, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-02T14:57:26.954 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:26 smithi169 ceph-mgr[26985]: -- 172.21.15.169:0/701779017 <== mon.2 v2:172.21.15.169:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55dd3bd76340 con 0x55dd3bd4ec00 2024-04-02T14:57:26.954 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:26 smithi169 bash[26966]: 794578fdbdb93f619994fc2d7cffce882fad4f6b6e5555cdb5140388171f4ef2 2024-04-02T14:57:27.228 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:26 smithi169 systemd[1]: Started Ceph mgr.x for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T14:57:27.685 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:27 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:27.391+0000 7f061d907000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T14:57:27.685 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:27 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:27.486+0000 7f061d907000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T14:57:27.773 INFO:teuthology.orchestra.run.smithi042.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-02T14:57:27.773 INFO:teuthology.orchestra.run.smithi042.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-02T14:57:27.773 INFO:teuthology.orchestra.run.smithi042.stderr: stderr: 10+0 records in 2024-04-02T14:57:27.773 INFO:teuthology.orchestra.run.smithi042.stderr:10+0 records out 2024-04-02T14:57:27.773 INFO:teuthology.orchestra.run.smithi042.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00846962 s, 1.2 GB/s 2024-04-02T14:57:27.773 INFO:teuthology.orchestra.run.smithi042.stderr:--> Zapping successful for: 2024-04-02T14:57:28.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:28.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:28.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:28.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:28.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[24805]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:28.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:28.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:28.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T14:57:28.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T14:57:28.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:28.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:28.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:28.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:28.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:28.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[29473]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:28.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:28.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:28.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T14:57:28.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T14:57:28.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:28 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:28.329 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi042:vg_nvme/lv_4 2024-04-02T14:57:28.405 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:28.098+0000 7f061d907000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T14:57:28.405 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:28.185+0000 7f061d907000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T14:57:28.405 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:28.266+0000 7f061d907000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T14:57:28.405 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:28.405 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:28.405 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:28.405 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:28.405 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-mon[25801]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:28.406 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:28.406 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:28.406 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T14:57:28.406 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T14:57:28.406 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:28.672 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:28.414+0000 7f061d907000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T14:57:28.672 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:28.493+0000 7f061d907000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T14:57:28.672 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:28.600+0000 7f061d907000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T14:57:28.672 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:28.682+0000 7f061d907000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T14:57:28.978 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:28.843+0000 7f061d907000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T14:57:28.978 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:28 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:28.927+0000 7f061d907000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T14:57:29.279 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:29 smithi042 ceph-mon[24805]: Reconfiguring mgr.y (unknown last config time)... 2024-04-02T14:57:29.279 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:29 smithi042 ceph-mon[24805]: Reconfiguring daemon mgr.y on smithi042 2024-04-02T14:57:29.279 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:29 smithi042 ceph-mon[29473]: Reconfiguring mgr.y (unknown last config time)... 2024-04-02T14:57:29.280 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:29 smithi042 ceph-mon[29473]: Reconfiguring daemon mgr.y on smithi042 2024-04-02T14:57:29.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:29 smithi169 ceph-mon[25801]: Reconfiguring mgr.y (unknown last config time)... 2024-04-02T14:57:29.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:29 smithi169 ceph-mon[25801]: Reconfiguring daemon mgr.y on smithi042 2024-04-02T14:57:30.229 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:29 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:29.958+0000 7f061d907000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T14:57:30.229 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:30 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:30.087+0000 7f061d907000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T14:57:30.561 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:30 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:30.276+0000 7f061d907000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T14:57:30.561 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:30 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:30.362+0000 7f061d907000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T14:57:30.561 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:30 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:30.496+0000 7f061d907000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T14:57:30.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:30 smithi042 ceph-mon[24805]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:30.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:30 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:30.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:30 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:30.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:30 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:30.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:30 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:30.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:30 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:30.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:30 smithi042 ceph-mon[29473]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:30.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:30 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:30.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:30 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:30.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:30 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:30.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:30 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:30.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:30 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:30.978 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:30 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:30.815+0000 7f061d907000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T14:57:30.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:30 smithi169 ceph-mon[25801]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:30.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:30 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:30.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:30 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:30.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:30 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:30.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:30 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:30.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:30 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:31.478 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:31.031+0000 7f061d907000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T14:57:31.479 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:31.115+0000 7f061d907000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T14:57:31.479 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:31.280+0000 7f061d907000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T14:57:31.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:31 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:57:31.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:31 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:57:31.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:31 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:31.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:31 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:57:31.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:31 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:31.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:31 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T14:57:32.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:57:32.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:57:32.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:32.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:57:32.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:32.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T14:57:32.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:31 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:57:32.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:31 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:57:32.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:31 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:32.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:31 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:57:32.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:31 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:32.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:31 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T14:57:32.470 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:32.190+0000 7f061d907000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T14:57:32.470 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:32.480+0000 7f061d907000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T14:57:32.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:32 smithi169 ceph-mon[25801]: from='client.24124 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:57:32.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:32 smithi169 ceph-mon[25801]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:32.729 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:32.572+0000 7f061d907000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T14:57:32.729 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 14:57:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T14:57:32.722+0000 7f061d907000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T14:57:32.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:32 smithi042 ceph-mon[24805]: from='client.24124 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:57:32.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:32 smithi042 ceph-mon[24805]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:32.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:32 smithi042 ceph-mon[29473]: from='client.24124 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:57:32.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:32 smithi042 ceph-mon[29473]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:33.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:33 smithi169 ceph-mon[25801]: Standby manager daemon x started 2024-04-02T14:57:33.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:33 smithi169 ceph-mon[25801]: from='mgr.? 172.21.15.169:0/1208208725' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T14:57:33.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:33 smithi169 ceph-mon[25801]: from='mgr.? 172.21.15.169:0/1208208725' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T14:57:33.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:33 smithi169 ceph-mon[25801]: from='mgr.? 172.21.15.169:0/1208208725' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T14:57:33.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:33 smithi169 ceph-mon[25801]: from='mgr.? 172.21.15.169:0/1208208725' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T14:57:33.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:33 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3328993339' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "177da1b0-ec41-4bbe-8d6b-988afdd4cbbc"}]: dispatch 2024-04-02T14:57:33.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:33 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3328993339' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "177da1b0-ec41-4bbe-8d6b-988afdd4cbbc"}]': finished 2024-04-02T14:57:33.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:33 smithi169 ceph-mon[25801]: osdmap e5: 1 total, 0 up, 1 in 2024-04-02T14:57:33.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:33 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:57:34.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[24805]: Standby manager daemon x started 2024-04-02T14:57:34.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[24805]: from='mgr.? 172.21.15.169:0/1208208725' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T14:57:34.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[24805]: from='mgr.? 172.21.15.169:0/1208208725' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T14:57:34.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[24805]: from='mgr.? 172.21.15.169:0/1208208725' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T14:57:34.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[24805]: from='mgr.? 172.21.15.169:0/1208208725' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T14:57:34.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3328993339' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "177da1b0-ec41-4bbe-8d6b-988afdd4cbbc"}]: dispatch 2024-04-02T14:57:34.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3328993339' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "177da1b0-ec41-4bbe-8d6b-988afdd4cbbc"}]': finished 2024-04-02T14:57:34.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[24805]: osdmap e5: 1 total, 0 up, 1 in 2024-04-02T14:57:34.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:57:34.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[29473]: Standby manager daemon x started 2024-04-02T14:57:34.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[29473]: from='mgr.? 172.21.15.169:0/1208208725' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T14:57:34.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[29473]: from='mgr.? 172.21.15.169:0/1208208725' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T14:57:34.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[29473]: from='mgr.? 172.21.15.169:0/1208208725' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T14:57:34.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[29473]: from='mgr.? 172.21.15.169:0/1208208725' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T14:57:34.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3328993339' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "177da1b0-ec41-4bbe-8d6b-988afdd4cbbc"}]: dispatch 2024-04-02T14:57:34.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3328993339' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "177da1b0-ec41-4bbe-8d6b-988afdd4cbbc"}]': finished 2024-04-02T14:57:34.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[29473]: osdmap e5: 1 total, 0 up, 1 in 2024-04-02T14:57:34.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:33 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:57:34.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:34 smithi169 ceph-mon[25801]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:34.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:34 smithi169 ceph-mon[25801]: mgrmap e14: y(active, since 63s), standbys: x 2024-04-02T14:57:34.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:34 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T14:57:34.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:34 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3089950430' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:57:35.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:34 smithi042 ceph-mon[24805]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:35.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:34 smithi042 ceph-mon[24805]: mgrmap e14: y(active, since 63s), standbys: x 2024-04-02T14:57:35.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T14:57:35.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:34 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3089950430' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:57:35.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:34 smithi042 ceph-mon[29473]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:35.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:34 smithi042 ceph-mon[29473]: mgrmap e14: y(active, since 63s), standbys: x 2024-04-02T14:57:35.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:34 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T14:57:35.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:34 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3089950430' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:57:36.383 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:36 smithi042 ceph-mon[24805]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:36.383 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:36 smithi042 ceph-mon[29473]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:36.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:36 smithi169 ceph-mon[25801]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:38.434 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:38 smithi042 ceph-mon[24805]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:38.435 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:38 smithi042 ceph-mon[29473]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:38.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:38 smithi169 ceph-mon[25801]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:40.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:40 smithi169 ceph-mon[25801]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:40.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:40 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T14:57:40.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:40 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:40.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:40 smithi169 ceph-mon[25801]: Deploying daemon osd.0 on smithi042 2024-04-02T14:57:40.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:40 smithi042 ceph-mon[29473]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:40.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:40 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T14:57:40.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:40 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:40.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:40 smithi042 ceph-mon[29473]: Deploying daemon osd.0 on smithi042 2024-04-02T14:57:40.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:40 smithi042 ceph-mon[24805]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:40.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:40 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T14:57:40.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:40 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:40.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:40 smithi042 ceph-mon[24805]: Deploying daemon osd.0 on smithi042 2024-04-02T14:57:42.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:42 smithi169 ceph-mon[25801]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:42.520 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:42 smithi042 ceph-mon[24805]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:42.521 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:42 smithi042 ceph-mon[29473]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:44.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:44 smithi042 ceph-mon[24805]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:44.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:44 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:44.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:44 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:44.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:44 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:44.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:44 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:44.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:44 smithi042 ceph-mon[29473]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:44.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:44 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:44.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:44 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:44.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:44 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:44.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:44 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:44.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:44 smithi169 ceph-mon[25801]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:44.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:44 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:44.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:44 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:44.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:44 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:44.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:44 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:46.303 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:46 smithi042 ceph-mon[24805]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:46.303 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:46 smithi042 ceph-mon[29473]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:46.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:46 smithi169 ceph-mon[25801]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:47.472 INFO:teuthology.orchestra.run.smithi042.stdout:Created osd(s) 0 on host 'smithi042' 2024-04-02T14:57:47.788 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:47.788 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:47.788 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[24805]: from='osd.0 [v2:172.21.15.42:6802/2441352487,v1:172.21.15.42:6803/2441352487]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T14:57:47.788 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[24805]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T14:57:47.788 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:47.788 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:47.788 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:47.788 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:47.789 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:47.789 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:47.789 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[29473]: from='osd.0 [v2:172.21.15.42:6802/2441352487,v1:172.21.15.42:6803/2441352487]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T14:57:47.789 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[29473]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T14:57:47.789 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:47.789 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:47.789 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:47.789 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:47 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:47.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:47 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:47.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:47 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:47.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:47 smithi169 ceph-mon[25801]: from='osd.0 [v2:172.21.15.42:6802/2441352487,v1:172.21.15.42:6803/2441352487]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T14:57:47.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:47 smithi169 ceph-mon[25801]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T14:57:47.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:47 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:47.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:47 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:57:47.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:47 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:47.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:47 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:57:48.077 DEBUG:teuthology.orchestra.run.smithi042:osd.0> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.0.service 2024-04-02T14:57:48.079 INFO:tasks.cephadm:Deploying osd.1 on smithi042 with /dev/vg_nvme/lv_3... 2024-04-02T14:57:48.080 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-02T14:57:48.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:48 smithi042 ceph-mon[29473]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:48.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:48 smithi042 ceph-mon[29473]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T14:57:48.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:48 smithi042 ceph-mon[29473]: osdmap e6: 1 total, 0 up, 1 in 2024-04-02T14:57:48.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:48 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:57:48.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:48 smithi042 ceph-mon[29473]: from='osd.0 [v2:172.21.15.42:6802/2441352487,v1:172.21.15.42:6803/2441352487]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:57:48.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:48 smithi042 ceph-mon[29473]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:57:48.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:48 smithi042 ceph-mon[24805]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:48.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:48 smithi042 ceph-mon[24805]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T14:57:48.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:48 smithi042 ceph-mon[24805]: osdmap e6: 1 total, 0 up, 1 in 2024-04-02T14:57:48.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:48 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:57:48.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:48 smithi042 ceph-mon[24805]: from='osd.0 [v2:172.21.15.42:6802/2441352487,v1:172.21.15.42:6803/2441352487]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:57:48.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:48 smithi042 ceph-mon[24805]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:57:48.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:48 smithi169 ceph-mon[25801]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:48.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:48 smithi169 ceph-mon[25801]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T14:57:48.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:48 smithi169 ceph-mon[25801]: osdmap e6: 1 total, 0 up, 1 in 2024-04-02T14:57:48.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:48 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:57:48.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:48 smithi169 ceph-mon[25801]: from='osd.0 [v2:172.21.15.42:6802/2441352487,v1:172.21.15.42:6803/2441352487]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:57:48.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:48 smithi169 ceph-mon[25801]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:57:49.294 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 14:57:48 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-0[32322]: 2024-04-02T14:57:48.932+0000 7fdaec3c5700 -1 osd.0 0 waiting for initial osdmap 2024-04-02T14:57:49.294 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 14:57:48 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-0[32322]: 2024-04-02T14:57:48.935+0000 7fdae3555700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T14:57:50.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:49 smithi042 ceph-mon[24805]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-04-02T14:57:50.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:49 smithi042 ceph-mon[24805]: osdmap e7: 1 total, 0 up, 1 in 2024-04-02T14:57:50.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:49 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:57:50.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:49 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:57:50.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:49 smithi042 ceph-mon[24805]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:50.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:49 smithi042 ceph-mon[29473]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-04-02T14:57:50.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:49 smithi042 ceph-mon[29473]: osdmap e7: 1 total, 0 up, 1 in 2024-04-02T14:57:50.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:49 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:57:50.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:49 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:57:50.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:49 smithi042 ceph-mon[29473]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:50.228 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:49 smithi169 ceph-mon[25801]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-04-02T14:57:50.228 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:49 smithi169 ceph-mon[25801]: osdmap e7: 1 total, 0 up, 1 in 2024-04-02T14:57:50.228 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:49 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:57:50.228 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:49 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:57:50.229 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:49 smithi169 ceph-mon[25801]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:57:51.134 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:50 smithi042 ceph-mon[24805]: purged_snaps scrub starts 2024-04-02T14:57:51.134 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:50 smithi042 ceph-mon[24805]: purged_snaps scrub ok 2024-04-02T14:57:51.135 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:50 smithi042 ceph-mon[24805]: osd.0 [v2:172.21.15.42:6802/2441352487,v1:172.21.15.42:6803/2441352487] boot 2024-04-02T14:57:51.135 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:50 smithi042 ceph-mon[24805]: osdmap e8: 1 total, 1 up, 1 in 2024-04-02T14:57:51.135 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:50 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:57:51.135 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:50 smithi042 ceph-mon[29473]: purged_snaps scrub starts 2024-04-02T14:57:51.135 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:50 smithi042 ceph-mon[29473]: purged_snaps scrub ok 2024-04-02T14:57:51.135 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:50 smithi042 ceph-mon[29473]: osd.0 [v2:172.21.15.42:6802/2441352487,v1:172.21.15.42:6803/2441352487] boot 2024-04-02T14:57:51.135 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:50 smithi042 ceph-mon[29473]: osdmap e8: 1 total, 1 up, 1 in 2024-04-02T14:57:51.135 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:50 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:57:51.228 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:50 smithi169 ceph-mon[25801]: purged_snaps scrub starts 2024-04-02T14:57:51.228 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:50 smithi169 ceph-mon[25801]: purged_snaps scrub ok 2024-04-02T14:57:51.229 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:50 smithi169 ceph-mon[25801]: osd.0 [v2:172.21.15.42:6802/2441352487,v1:172.21.15.42:6803/2441352487] boot 2024-04-02T14:57:51.229 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:50 smithi169 ceph-mon[25801]: osdmap e8: 1 total, 1 up, 1 in 2024-04-02T14:57:51.229 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:50 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:57:51.986 INFO:teuthology.orchestra.run.smithi042.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-02T14:57:51.986 INFO:teuthology.orchestra.run.smithi042.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-02T14:57:51.986 INFO:teuthology.orchestra.run.smithi042.stderr: stderr: 10+0 records in 2024-04-02T14:57:51.986 INFO:teuthology.orchestra.run.smithi042.stderr:10+0 records out 2024-04-02T14:57:51.986 INFO:teuthology.orchestra.run.smithi042.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00876612 s, 1.2 GB/s 2024-04-02T14:57:51.986 INFO:teuthology.orchestra.run.smithi042.stderr:--> Zapping successful for: 2024-04-02T14:57:52.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[29473]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:57:52.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[29473]: osdmap e9: 1 total, 1 up, 1 in 2024-04-02T14:57:52.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[29473]: Detected new or changed devices on smithi042 2024-04-02T14:57:52.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:52.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:57:52.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[29473]: Adjusting osd_memory_target on smithi042 to 16069M 2024-04-02T14:57:52.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:52.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:52.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[24805]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:57:52.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[24805]: osdmap e9: 1 total, 1 up, 1 in 2024-04-02T14:57:52.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[24805]: Detected new or changed devices on smithi042 2024-04-02T14:57:52.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:52.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:57:52.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[24805]: Adjusting osd_memory_target on smithi042 to 16069M 2024-04-02T14:57:52.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:52.296 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:52 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:52.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:52 smithi169 ceph-mon[25801]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:57:52.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:52 smithi169 ceph-mon[25801]: osdmap e9: 1 total, 1 up, 1 in 2024-04-02T14:57:52.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:52 smithi169 ceph-mon[25801]: Detected new or changed devices on smithi042 2024-04-02T14:57:52.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:52 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:52.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:52 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:57:52.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:52 smithi169 ceph-mon[25801]: Adjusting osd_memory_target on smithi042 to 16069M 2024-04-02T14:57:52.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:52 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:52.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:52 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:57:52.616 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi042:vg_nvme/lv_3 2024-04-02T14:57:54.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:54 smithi169 ceph-mon[25801]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:57:54.498 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:54 smithi042 ceph-mon[24805]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:57:54.498 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:54 smithi042 ceph-mon[29473]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:57:55.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:55 smithi169 ceph-mon[25801]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:57:55.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:55 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:57:55.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:55 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:57:55.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:55 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:55.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:55 smithi042 ceph-mon[29473]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:57:55.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:55 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:57:55.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:55 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:57:55.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:55 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:55.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:55 smithi042 ceph-mon[24805]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:57:55.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:55 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:57:55.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:55 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:57:55.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:55 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:57:56.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:56 smithi042 ceph-mon[24805]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:57:56.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:56 smithi042 ceph-mon[29473]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:57:56.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:56 smithi169 ceph-mon[25801]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:57:57.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:57 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/2086667805' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aac29b7a-cf39-4dee-87e3-a905551d6230"}]: dispatch 2024-04-02T14:57:57.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:57 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/2086667805' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aac29b7a-cf39-4dee-87e3-a905551d6230"}]': finished 2024-04-02T14:57:57.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:57 smithi169 ceph-mon[25801]: osdmap e10: 2 total, 1 up, 2 in 2024-04-02T14:57:57.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:57 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:57:57.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:57 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/2086667805' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aac29b7a-cf39-4dee-87e3-a905551d6230"}]: dispatch 2024-04-02T14:57:57.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:57 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/2086667805' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aac29b7a-cf39-4dee-87e3-a905551d6230"}]': finished 2024-04-02T14:57:57.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:57 smithi042 ceph-mon[24805]: osdmap e10: 2 total, 1 up, 2 in 2024-04-02T14:57:57.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:57 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:57:57.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:57 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/2086667805' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aac29b7a-cf39-4dee-87e3-a905551d6230"}]: dispatch 2024-04-02T14:57:57.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:57 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/2086667805' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aac29b7a-cf39-4dee-87e3-a905551d6230"}]': finished 2024-04-02T14:57:57.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:57 smithi042 ceph-mon[29473]: osdmap e10: 2 total, 1 up, 2 in 2024-04-02T14:57:57.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:57 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:57:58.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:58 smithi169 ceph-mon[25801]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:57:58.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:57:58 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1568547482' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:57:58.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:58 smithi042 ceph-mon[29473]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:57:58.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:57:58 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1568547482' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:57:58.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:58 smithi042 ceph-mon[24805]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:57:58.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:57:58 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1568547482' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:58:00.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:00 smithi169 ceph-mon[25801]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:00.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:00 smithi042 ceph-mon[24805]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:00.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:00 smithi042 ceph-mon[29473]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:02.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:02 smithi169 ceph-mon[25801]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:02.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:02 smithi042 ceph-mon[24805]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:02.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:02 smithi042 ceph-mon[29473]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:03.457 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T14:58:03.457 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:03.458 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T14:58:03.458 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:03.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T14:58:03.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:04.463 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:04 smithi042 ceph-mon[24805]: Deploying daemon osd.1 on smithi042 2024-04-02T14:58:04.463 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:04 smithi042 ceph-mon[24805]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:04.464 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:04 smithi042 ceph-mon[29473]: Deploying daemon osd.1 on smithi042 2024-04-02T14:58:04.464 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:04 smithi042 ceph-mon[29473]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:04.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:04 smithi169 ceph-mon[25801]: Deploying daemon osd.1 on smithi042 2024-04-02T14:58:04.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:04 smithi169 ceph-mon[25801]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:06.424 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:06 smithi042 ceph-mon[24805]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:06.424 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:06 smithi042 ceph-mon[29473]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:06.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:06 smithi169 ceph-mon[25801]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:08.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:08 smithi042 ceph-mon[29473]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:08.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:08 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:08.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:08 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:58:08.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:08 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:08.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:08 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:58:08.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:08 smithi042 ceph-mon[24805]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:08.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:08 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:08.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:08 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:58:08.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:08 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:08.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:08 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:58:08.297 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:08 smithi169 ceph-mon[25801]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:08.297 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:08 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:08.297 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:08 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:58:08.297 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:08 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:08.297 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:08 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:58:10.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:10 smithi169 ceph-mon[25801]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:10.502 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:10 smithi042 ceph-mon[29473]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:10.502 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:10 smithi042 ceph-mon[24805]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:11.289 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:11 smithi042 ceph-mon[24805]: from='osd.1 [v2:172.21.15.42:6810/146974053,v1:172.21.15.42:6811/146974053]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T14:58:11.289 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:11 smithi042 ceph-mon[24805]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T14:58:11.289 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:11 smithi042 ceph-mon[29473]: from='osd.1 [v2:172.21.15.42:6810/146974053,v1:172.21.15.42:6811/146974053]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T14:58:11.289 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:11 smithi042 ceph-mon[29473]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T14:58:11.321 INFO:teuthology.orchestra.run.smithi042.stdout:Created osd(s) 1 on host 'smithi042' 2024-04-02T14:58:11.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:11 smithi169 ceph-mon[25801]: from='osd.1 [v2:172.21.15.42:6810/146974053,v1:172.21.15.42:6811/146974053]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T14:58:11.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:11 smithi169 ceph-mon[25801]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T14:58:11.981 DEBUG:teuthology.orchestra.run.smithi042:osd.1> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.1.service 2024-04-02T14:58:11.984 INFO:tasks.cephadm:Deploying osd.2 on smithi042 with /dev/vg_nvme/lv_2... 2024-04-02T14:58:11.984 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-02T14:58:12.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[29473]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:12.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[29473]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T14:58:12.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[29473]: osdmap e11: 2 total, 1 up, 2 in 2024-04-02T14:58:12.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:58:12.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[29473]: from='osd.1 [v2:172.21.15.42:6810/146974053,v1:172.21.15.42:6811/146974053]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:58:12.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[29473]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:58:12.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:12.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:12.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[24805]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:12.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[24805]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T14:58:12.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[24805]: osdmap e11: 2 total, 1 up, 2 in 2024-04-02T14:58:12.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:58:12.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[24805]: from='osd.1 [v2:172.21.15.42:6810/146974053,v1:172.21.15.42:6811/146974053]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:58:12.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[24805]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:58:12.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:12.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:12.296 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-1[35300]: 2024-04-02T14:58:12.214+0000 7f082c542700 -1 osd.1 0 waiting for initial osdmap 2024-04-02T14:58:12.296 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 14:58:12 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-1[35300]: 2024-04-02T14:58:12.217+0000 7f0826ed9700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T14:58:12.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:12 smithi169 ceph-mon[25801]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:12.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:12 smithi169 ceph-mon[25801]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T14:58:12.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:12 smithi169 ceph-mon[25801]: osdmap e11: 2 total, 1 up, 2 in 2024-04-02T14:58:12.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:12 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:58:12.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:12 smithi169 ceph-mon[25801]: from='osd.1 [v2:172.21.15.42:6810/146974053,v1:172.21.15.42:6811/146974053]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:58:12.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:12 smithi169 ceph-mon[25801]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:58:12.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:12 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:12.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:12 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:13.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:13 smithi042 ceph-mon[24805]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-04-02T14:58:13.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:13 smithi042 ceph-mon[24805]: osdmap e12: 2 total, 1 up, 2 in 2024-04-02T14:58:13.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:13 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:58:13.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:13 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:58:13.291 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:13 smithi042 ceph-mon[29473]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-04-02T14:58:13.291 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:13 smithi042 ceph-mon[29473]: osdmap e12: 2 total, 1 up, 2 in 2024-04-02T14:58:13.291 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:13 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:58:13.291 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:13 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:58:13.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:13 smithi169 ceph-mon[25801]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-04-02T14:58:13.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:13 smithi169 ceph-mon[25801]: osdmap e12: 2 total, 1 up, 2 in 2024-04-02T14:58:13.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:13 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:58:13.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:13 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:58:14.010 INFO:teuthology.orchestra.run.smithi042.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-02T14:58:14.010 INFO:teuthology.orchestra.run.smithi042.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-02T14:58:14.010 INFO:teuthology.orchestra.run.smithi042.stderr: stderr: 10+0 records in 2024-04-02T14:58:14.010 INFO:teuthology.orchestra.run.smithi042.stderr:10+0 records out 2024-04-02T14:58:14.010 INFO:teuthology.orchestra.run.smithi042.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0096983 s, 1.1 GB/s 2024-04-02T14:58:14.010 INFO:teuthology.orchestra.run.smithi042.stderr:--> Zapping successful for: 2024-04-02T14:58:14.240 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:14 smithi042 ceph-mon[24805]: purged_snaps scrub starts 2024-04-02T14:58:14.240 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:14 smithi042 ceph-mon[24805]: purged_snaps scrub ok 2024-04-02T14:58:14.241 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:14 smithi042 ceph-mon[24805]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:14.241 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:14 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:58:14.241 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:14 smithi042 ceph-mon[24805]: osd.1 [v2:172.21.15.42:6810/146974053,v1:172.21.15.42:6811/146974053] boot 2024-04-02T14:58:14.241 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:14 smithi042 ceph-mon[24805]: osdmap e13: 2 total, 2 up, 2 in 2024-04-02T14:58:14.241 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:14 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:58:14.241 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:14 smithi042 ceph-mon[29473]: purged_snaps scrub starts 2024-04-02T14:58:14.241 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:14 smithi042 ceph-mon[29473]: purged_snaps scrub ok 2024-04-02T14:58:14.241 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:14 smithi042 ceph-mon[29473]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:14.241 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:14 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:58:14.241 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:14 smithi042 ceph-mon[29473]: osd.1 [v2:172.21.15.42:6810/146974053,v1:172.21.15.42:6811/146974053] boot 2024-04-02T14:58:14.241 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:14 smithi042 ceph-mon[29473]: osdmap e13: 2 total, 2 up, 2 in 2024-04-02T14:58:14.242 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:14 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:58:14.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:14 smithi169 ceph-mon[25801]: purged_snaps scrub starts 2024-04-02T14:58:14.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:14 smithi169 ceph-mon[25801]: purged_snaps scrub ok 2024-04-02T14:58:14.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:14 smithi169 ceph-mon[25801]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:58:14.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:14 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:58:14.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:14 smithi169 ceph-mon[25801]: osd.1 [v2:172.21.15.42:6810/146974053,v1:172.21.15.42:6811/146974053] boot 2024-04-02T14:58:14.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:14 smithi169 ceph-mon[25801]: osdmap e13: 2 total, 2 up, 2 in 2024-04-02T14:58:14.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:14 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:58:15.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:15 smithi169 ceph-mon[25801]: osdmap e14: 2 total, 2 up, 2 in 2024-04-02T14:58:15.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:15 smithi042 ceph-mon[29473]: osdmap e14: 2 total, 2 up, 2 in 2024-04-02T14:58:15.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:15 smithi042 ceph-mon[24805]: osdmap e14: 2 total, 2 up, 2 in 2024-04-02T14:58:15.572 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi042:vg_nvme/lv_2 2024-04-02T14:58:16.470 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:16 smithi042 ceph-mon[24805]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:16.470 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:16 smithi042 ceph-mon[29473]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:16.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:16 smithi169 ceph-mon[25801]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:18.375 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[29473]: Detected new or changed devices on smithi042 2024-04-02T14:58:18.375 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:18.375 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:58:18.375 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:58:18.376 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[29473]: Adjusting osd_memory_target on smithi042 to 8034M 2024-04-02T14:58:18.376 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:18.376 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:18.376 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:58:18.376 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:18.376 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:58:18.376 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:18.376 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[29473]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:18.376 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[24805]: Detected new or changed devices on smithi042 2024-04-02T14:58:18.376 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:18.376 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:58:18.376 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:58:18.377 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[24805]: Adjusting osd_memory_target on smithi042 to 8034M 2024-04-02T14:58:18.377 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:18.377 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:18.377 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:58:18.377 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:18.377 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:58:18.377 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:18.377 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:18 smithi042 ceph-mon[24805]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:18.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:18 smithi169 ceph-mon[25801]: Detected new or changed devices on smithi042 2024-04-02T14:58:18.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:18.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:58:18.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:58:18.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:18 smithi169 ceph-mon[25801]: Adjusting osd_memory_target on smithi042 to 8034M 2024-04-02T14:58:18.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:18.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:18.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:58:18.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:18.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:58:18.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:18 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:18.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:18 smithi169 ceph-mon[25801]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:19.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:19 smithi042 ceph-mon[24805]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:58:19.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:19 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:58:19.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:19 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:58:19.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:19 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:19.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:19 smithi042 ceph-mon[29473]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:58:19.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:19 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:58:19.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:19 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:58:19.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:19 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:19.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:19 smithi169 ceph-mon[25801]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:58:19.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:19 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:58:19.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:19 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:58:19.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:19 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:20.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:20 smithi169 ceph-mon[25801]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:20.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:20 smithi042 ceph-mon[24805]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:20.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:20 smithi042 ceph-mon[29473]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:21.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:21 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3203906513' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "710ff1f2-d67f-405c-b48b-64e9fd8b6b68"}]: dispatch 2024-04-02T14:58:21.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:21 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3203906513' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "710ff1f2-d67f-405c-b48b-64e9fd8b6b68"}]': finished 2024-04-02T14:58:21.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:21 smithi169 ceph-mon[25801]: osdmap e15: 3 total, 2 up, 3 in 2024-04-02T14:58:21.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:21 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:58:21.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:21 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/993578752' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:58:21.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:21 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3203906513' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "710ff1f2-d67f-405c-b48b-64e9fd8b6b68"}]: dispatch 2024-04-02T14:58:21.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:21 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3203906513' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "710ff1f2-d67f-405c-b48b-64e9fd8b6b68"}]': finished 2024-04-02T14:58:21.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:21 smithi042 ceph-mon[24805]: osdmap e15: 3 total, 2 up, 3 in 2024-04-02T14:58:21.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:21 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:58:21.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:21 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/993578752' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:58:21.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:21 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3203906513' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "710ff1f2-d67f-405c-b48b-64e9fd8b6b68"}]: dispatch 2024-04-02T14:58:21.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:21 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3203906513' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "710ff1f2-d67f-405c-b48b-64e9fd8b6b68"}]': finished 2024-04-02T14:58:21.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:21 smithi042 ceph-mon[29473]: osdmap e15: 3 total, 2 up, 3 in 2024-04-02T14:58:21.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:21 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:58:21.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:21 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/993578752' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:58:22.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:22 smithi169 ceph-mon[25801]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:22.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:22 smithi042 ceph-mon[24805]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:22.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:22 smithi042 ceph-mon[29473]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:24.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:24 smithi042 ceph-mon[24805]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:24.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:24 smithi042 ceph-mon[29473]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:24.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:24 smithi169 ceph-mon[25801]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:26.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:26 smithi042 ceph-mon[24805]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:26.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:26 smithi042 ceph-mon[29473]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:26.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:26 smithi169 ceph-mon[25801]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:27.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:27 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T14:58:27.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:27 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:27.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:27 smithi169 ceph-mon[25801]: Deploying daemon osd.2 on smithi042 2024-04-02T14:58:27.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:27 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T14:58:27.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:27 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:27.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:27 smithi042 ceph-mon[24805]: Deploying daemon osd.2 on smithi042 2024-04-02T14:58:27.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:27 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T14:58:27.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:27 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:27.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:27 smithi042 ceph-mon[29473]: Deploying daemon osd.2 on smithi042 2024-04-02T14:58:28.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:28 smithi169 ceph-mon[25801]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:28.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:28 smithi042 ceph-mon[24805]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:28.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:28 smithi042 ceph-mon[29473]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:30.713 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:30 smithi042 ceph-mon[24805]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:30.713 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:30 smithi042 ceph-mon[29473]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:30.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:30 smithi169 ceph-mon[25801]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:31.604 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:58:31.604 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T14:58:31.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:31 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:58:31.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:31 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T14:58:31.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:31 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:58:31.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:31 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T14:58:32.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:32 smithi042 ceph-mon[24805]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:32.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:32 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:32.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:32 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:58:32.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:32 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:32.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:32 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:58:32.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:32 smithi042 ceph-mon[29473]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:32.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:32 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:32.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:32 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:58:32.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:32 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:32.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:32 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:58:32.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:32 smithi169 ceph-mon[25801]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:32.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:32 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:32.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:32 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:58:32.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:32 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:32.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:32 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:58:34.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:34 smithi042 ceph-mon[29473]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:34.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:34 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:34.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:34 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:34.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:34 smithi042 ceph-mon[24805]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:34.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:34.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:34.934 INFO:teuthology.orchestra.run.smithi042.stdout:Created osd(s) 2 on host 'smithi042' 2024-04-02T14:58:34.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:34 smithi169 ceph-mon[25801]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:34.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:34 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:34.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:34 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:35.595 DEBUG:teuthology.orchestra.run.smithi042:osd.2> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.2.service 2024-04-02T14:58:35.598 INFO:tasks.cephadm:Deploying osd.3 on smithi042 with /dev/vg_nvme/lv_1... 2024-04-02T14:58:35.598 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-02T14:58:35.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:35 smithi042 ceph-mon[24805]: from='osd.2 [v2:172.21.15.42:6818/744264554,v1:172.21.15.42:6819/744264554]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T14:58:35.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:35 smithi042 ceph-mon[24805]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T14:58:35.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:35 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:35.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:35 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:58:35.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:35 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:35.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:35 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:58:35.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:35 smithi042 ceph-mon[29473]: from='osd.2 [v2:172.21.15.42:6818/744264554,v1:172.21.15.42:6819/744264554]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T14:58:35.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:35 smithi042 ceph-mon[29473]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T14:58:35.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:35 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:35.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:35 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:58:35.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:35 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:35.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:35 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:58:35.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:35 smithi169 ceph-mon[25801]: from='osd.2 [v2:172.21.15.42:6818/744264554,v1:172.21.15.42:6819/744264554]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T14:58:35.983 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:35 smithi169 ceph-mon[25801]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T14:58:35.983 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:35 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:35.983 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:35 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:58:35.983 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:35 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:35.983 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:35 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:58:36.735 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:36 smithi042 ceph-mon[24805]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:36.736 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:36 smithi042 ceph-mon[24805]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T14:58:36.736 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:36 smithi042 ceph-mon[24805]: osdmap e16: 3 total, 2 up, 3 in 2024-04-02T14:58:36.736 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:36 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:58:36.736 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:36 smithi042 ceph-mon[24805]: from='osd.2 [v2:172.21.15.42:6818/744264554,v1:172.21.15.42:6819/744264554]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:58:36.736 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:36 smithi042 ceph-mon[24805]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:58:36.736 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:36 smithi042 ceph-mon[29473]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:36.736 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:36 smithi042 ceph-mon[29473]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T14:58:36.736 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:36 smithi042 ceph-mon[29473]: osdmap e16: 3 total, 2 up, 3 in 2024-04-02T14:58:36.737 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:36 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:58:36.737 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:36 smithi042 ceph-mon[29473]: from='osd.2 [v2:172.21.15.42:6818/744264554,v1:172.21.15.42:6819/744264554]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:58:36.737 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:36 smithi042 ceph-mon[29473]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:58:36.737 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 14:58:36 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-2[38215]: 2024-04-02T14:58:36.562+0000 7f64a3715700 -1 osd.2 0 waiting for initial osdmap 2024-04-02T14:58:36.737 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 14:58:36 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-2[38215]: 2024-04-02T14:58:36.565+0000 7f649d0aa700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T14:58:36.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:36 smithi169 ceph-mon[25801]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:36.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:36 smithi169 ceph-mon[25801]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T14:58:36.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:36 smithi169 ceph-mon[25801]: osdmap e16: 3 total, 2 up, 3 in 2024-04-02T14:58:36.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:36 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:58:36.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:36 smithi169 ceph-mon[25801]: from='osd.2 [v2:172.21.15.42:6818/744264554,v1:172.21.15.42:6819/744264554]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:58:36.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:36 smithi169 ceph-mon[25801]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:58:37.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:37 smithi042 ceph-mon[29473]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-04-02T14:58:37.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:37 smithi042 ceph-mon[29473]: osdmap e17: 3 total, 2 up, 3 in 2024-04-02T14:58:37.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:37 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:58:37.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:37 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:58:37.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:37 smithi042 ceph-mon[24805]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-04-02T14:58:37.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:37 smithi042 ceph-mon[24805]: osdmap e17: 3 total, 2 up, 3 in 2024-04-02T14:58:37.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:37 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:58:37.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:37 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:58:37.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:37 smithi169 ceph-mon[25801]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-04-02T14:58:37.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:37 smithi169 ceph-mon[25801]: osdmap e17: 3 total, 2 up, 3 in 2024-04-02T14:58:37.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:37 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:58:37.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:37 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:58:38.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:38 smithi042 ceph-mon[24805]: purged_snaps scrub starts 2024-04-02T14:58:38.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:38 smithi042 ceph-mon[24805]: purged_snaps scrub ok 2024-04-02T14:58:38.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:38 smithi042 ceph-mon[24805]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:38.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:38 smithi042 ceph-mon[24805]: osd.2 [v2:172.21.15.42:6818/744264554,v1:172.21.15.42:6819/744264554] boot 2024-04-02T14:58:38.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:38 smithi042 ceph-mon[24805]: osdmap e18: 3 total, 3 up, 3 in 2024-04-02T14:58:38.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:38 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:58:38.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:38 smithi042 ceph-mon[29473]: purged_snaps scrub starts 2024-04-02T14:58:38.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:38 smithi042 ceph-mon[29473]: purged_snaps scrub ok 2024-04-02T14:58:38.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:38 smithi042 ceph-mon[29473]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:38.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:38 smithi042 ceph-mon[29473]: osd.2 [v2:172.21.15.42:6818/744264554,v1:172.21.15.42:6819/744264554] boot 2024-04-02T14:58:38.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:38 smithi042 ceph-mon[29473]: osdmap e18: 3 total, 3 up, 3 in 2024-04-02T14:58:38.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:38 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:58:38.873 INFO:teuthology.orchestra.run.smithi042.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-02T14:58:38.873 INFO:teuthology.orchestra.run.smithi042.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-02T14:58:38.873 INFO:teuthology.orchestra.run.smithi042.stderr: stderr: 10+0 records in 2024-04-02T14:58:38.873 INFO:teuthology.orchestra.run.smithi042.stderr:10+0 records out 2024-04-02T14:58:38.873 INFO:teuthology.orchestra.run.smithi042.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00959875 s, 1.1 GB/s 2024-04-02T14:58:38.873 INFO:teuthology.orchestra.run.smithi042.stderr:--> Zapping successful for: 2024-04-02T14:58:38.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:38 smithi169 ceph-mon[25801]: purged_snaps scrub starts 2024-04-02T14:58:38.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:38 smithi169 ceph-mon[25801]: purged_snaps scrub ok 2024-04-02T14:58:38.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:38 smithi169 ceph-mon[25801]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:58:38.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:38 smithi169 ceph-mon[25801]: osd.2 [v2:172.21.15.42:6818/744264554,v1:172.21.15.42:6819/744264554] boot 2024-04-02T14:58:38.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:38 smithi169 ceph-mon[25801]: osdmap e18: 3 total, 3 up, 3 in 2024-04-02T14:58:38.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:38 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:58:39.836 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:39 smithi042 ceph-mon[24805]: osdmap e19: 3 total, 3 up, 3 in 2024-04-02T14:58:39.836 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:39 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-02T14:58:39.836 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:39 smithi042 ceph-mon[29473]: osdmap e19: 3 total, 3 up, 3 in 2024-04-02T14:58:39.837 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:39 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-02T14:58:39.860 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi042:vg_nvme/lv_1 2024-04-02T14:58:39.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:39 smithi169 ceph-mon[25801]: osdmap e19: 3 total, 3 up, 3 in 2024-04-02T14:58:39.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:39 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-02T14:58:40.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:40 smithi169 ceph-mon[25801]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:40.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:40 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-02T14:58:40.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:40 smithi169 ceph-mon[25801]: osdmap e20: 3 total, 3 up, 3 in 2024-04-02T14:58:40.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:40 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-02T14:58:40.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:40 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:40.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:40 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:58:40.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:40 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:58:40.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:40 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:58:40.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:40 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:40.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:40 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:41.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[29473]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:41.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-02T14:58:41.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[29473]: osdmap e20: 3 total, 3 up, 3 in 2024-04-02T14:58:41.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-02T14:58:41.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:41.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:58:41.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:58:41.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:58:41.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:41.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:41.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[24805]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:41.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-02T14:58:41.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[24805]: osdmap e20: 3 total, 3 up, 3 in 2024-04-02T14:58:41.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-02T14:58:41.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:41.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:58:41.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:58:41.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:58:41.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:41.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:40 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:41.391 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 14:58:41 smithi042 sudo[39512]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T14:58:41.392 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 14:58:41 smithi042 sudo[39512]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:58:41.392 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 14:58:41 smithi042 sudo[39512]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:58:41.674 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 14:58:41 smithi042 sudo[39515]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T14:58:41.674 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 14:58:41 smithi042 sudo[39515]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:58:41.674 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 14:58:41 smithi042 sudo[39515]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:58:41.674 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:41 smithi042 ceph-mon[24805]: Detected new or changed devices on smithi042 2024-04-02T14:58:41.674 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:41 smithi042 ceph-mon[24805]: Adjusting osd_memory_target on smithi042 to 5356M 2024-04-02T14:58:41.675 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:41 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-02T14:58:41.675 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:41 smithi042 ceph-mon[24805]: osdmap e21: 3 total, 3 up, 3 in 2024-04-02T14:58:41.675 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:41 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:41.675 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:41 smithi042 ceph-mon[29473]: Detected new or changed devices on smithi042 2024-04-02T14:58:41.675 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:41 smithi042 ceph-mon[29473]: Adjusting osd_memory_target on smithi042 to 5356M 2024-04-02T14:58:41.675 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:41 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-02T14:58:41.675 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:41 smithi042 ceph-mon[29473]: osdmap e21: 3 total, 3 up, 3 in 2024-04-02T14:58:41.675 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:41 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:41.944 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 14:58:41 smithi042 sudo[39537]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T14:58:41.944 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 14:58:41 smithi042 sudo[39537]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:58:41.944 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 14:58:41 smithi042 sudo[39537]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:58:41.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:41 smithi169 ceph-mon[25801]: Detected new or changed devices on smithi042 2024-04-02T14:58:41.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:41 smithi169 ceph-mon[25801]: Adjusting osd_memory_target on smithi042 to 5356M 2024-04-02T14:58:41.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:41 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-02T14:58:41.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:41 smithi169 ceph-mon[25801]: osdmap e21: 3 total, 3 up, 3 in 2024-04-02T14:58:41.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:41 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:42.202 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 14:58:41 smithi042 sudo[39566]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T14:58:42.202 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 14:58:41 smithi042 sudo[39566]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:58:42.203 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 14:58:41 smithi042 sudo[39566]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:58:42.467 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 14:58:42 smithi042 sudo[39606]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T14:58:42.467 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 14:58:42 smithi042 sudo[39606]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:58:42.467 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 14:58:42 smithi042 sudo[39606]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:58:42.726 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 14:58:42 smithi042 sudo[39649]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T14:58:42.726 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 14:58:42 smithi042 sudo[39649]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:58:42.726 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 14:58:42 smithi042 sudo[39649]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:58:42.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:42 smithi042 sudo[39670]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T14:58:42.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:42 smithi042 sudo[39670]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:58:42.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:42 smithi042 sudo[39670]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:58:43.256 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:43 smithi042 ceph-mon[29473]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:43.256 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:43 smithi042 ceph-mon[29473]: osdmap e22: 3 total, 3 up, 3 in 2024-04-02T14:58:43.256 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:43 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:58:43.256 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:43 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:58:43.256 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:43 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:43.256 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:43 smithi042 ceph-mon[29473]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T14:58:43.257 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:43 smithi042 sudo[39673]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T14:58:43.257 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:43 smithi042 sudo[39673]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:58:43.257 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:43 smithi042 sudo[39673]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:58:43.257 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:43 smithi042 ceph-mon[24805]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:43.257 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:43 smithi042 ceph-mon[24805]: osdmap e22: 3 total, 3 up, 3 in 2024-04-02T14:58:43.257 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:43 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:58:43.257 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:43 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:58:43.257 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:43 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:43.257 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:43 smithi042 ceph-mon[24805]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T14:58:43.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:43 smithi169 ceph-mon[25801]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:43.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:43 smithi169 ceph-mon[25801]: osdmap e22: 3 total, 3 up, 3 in 2024-04-02T14:58:43.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:43 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:58:43.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:43 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:58:43.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:43 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:43.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:43 smithi169 ceph-mon[25801]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T14:58:43.517 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:43 smithi042 sudo[39679]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T14:58:43.517 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:43 smithi042 sudo[39679]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:58:43.517 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:43 smithi042 sudo[39679]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:58:43.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:43 smithi042 sudo[39701]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T14:58:43.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:43 smithi042 sudo[39701]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:58:43.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:43 smithi042 sudo[39701]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:58:44.015 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:43 smithi169 sudo[27320]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T14:58:44.015 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:43 smithi169 sudo[27320]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:58:44.015 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:43 smithi169 sudo[27320]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:58:44.016 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 sudo[27323]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T14:58:44.016 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 sudo[27323]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:58:44.016 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 sudo[27323]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:58:44.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[24805]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:58:44.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[24805]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T14:58:44.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:58:44.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:58:44.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:58:44.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[24805]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T14:58:44.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:58:44.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:58:44.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:58:44.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[24805]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:44.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[24805]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T14:58:44.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:58:44.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:58:44.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:58:44.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[29473]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:58:44.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[29473]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T14:58:44.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:58:44.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:58:44.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:58:44.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[29473]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T14:58:44.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:58:44.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:58:44.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:58:44.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[29473]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:44.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[29473]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T14:58:44.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:58:44.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:58:44.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:44 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:58:44.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 ceph-mon[25801]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:58:44.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 ceph-mon[25801]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T14:58:44.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:58:44.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:58:44.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:58:44.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 ceph-mon[25801]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T14:58:44.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:58:44.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:58:44.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:58:44.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 ceph-mon[25801]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:44.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 ceph-mon[25801]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T14:58:44.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T14:58:44.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T14:58:44.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:44 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T14:58:45.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[24805]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T14:58:45.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[24805]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T14:58:45.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[24805]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-02T14:58:45.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:45.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1614897045' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27d9b9d7-e1f8-4be8-9da8-13dd4da8dad8"}]: dispatch 2024-04-02T14:58:45.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[24805]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27d9b9d7-e1f8-4be8-9da8-13dd4da8dad8"}]: dispatch 2024-04-02T14:58:45.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[24805]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "27d9b9d7-e1f8-4be8-9da8-13dd4da8dad8"}]': finished 2024-04-02T14:58:45.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[24805]: osdmap e23: 4 total, 3 up, 4 in 2024-04-02T14:58:45.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:58:45.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/537512138' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:58:45.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[29473]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T14:58:45.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[29473]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T14:58:45.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[29473]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-02T14:58:45.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:45.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1614897045' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27d9b9d7-e1f8-4be8-9da8-13dd4da8dad8"}]: dispatch 2024-04-02T14:58:45.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[29473]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27d9b9d7-e1f8-4be8-9da8-13dd4da8dad8"}]: dispatch 2024-04-02T14:58:45.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[29473]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "27d9b9d7-e1f8-4be8-9da8-13dd4da8dad8"}]': finished 2024-04-02T14:58:45.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[29473]: osdmap e23: 4 total, 3 up, 4 in 2024-04-02T14:58:45.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:58:45.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:45 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/537512138' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:58:45.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:45 smithi169 ceph-mon[25801]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T14:58:45.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:45 smithi169 ceph-mon[25801]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T14:58:45.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:45 smithi169 ceph-mon[25801]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-02T14:58:45.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:45 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:45.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:45 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1614897045' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27d9b9d7-e1f8-4be8-9da8-13dd4da8dad8"}]: dispatch 2024-04-02T14:58:45.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:45 smithi169 ceph-mon[25801]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27d9b9d7-e1f8-4be8-9da8-13dd4da8dad8"}]: dispatch 2024-04-02T14:58:45.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:45 smithi169 ceph-mon[25801]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "27d9b9d7-e1f8-4be8-9da8-13dd4da8dad8"}]': finished 2024-04-02T14:58:45.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:45 smithi169 ceph-mon[25801]: osdmap e23: 4 total, 3 up, 4 in 2024-04-02T14:58:45.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:45 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:58:45.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:45 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/537512138' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:58:46.443 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:46 smithi042 ceph-mon[24805]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:46.443 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:46 smithi042 ceph-mon[29473]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:46.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:46 smithi169 ceph-mon[25801]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:48.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:48 smithi169 ceph-mon[25801]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:48.518 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:48 smithi042 ceph-mon[24805]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:48.518 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:48 smithi042 ceph-mon[29473]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:50.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:50 smithi169 ceph-mon[25801]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:50.523 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:50 smithi042 ceph-mon[24805]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:50.523 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:50 smithi042 ceph-mon[29473]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:51.449 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:51 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T14:58:51.449 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:51 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:51.449 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:51 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T14:58:51.449 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:51 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:51.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:51 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T14:58:51.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:51 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:52.338 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:52 smithi042 ceph-mon[24805]: Deploying daemon osd.3 on smithi042 2024-04-02T14:58:52.338 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:52 smithi042 ceph-mon[24805]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:52.338 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:52 smithi042 ceph-mon[29473]: Deploying daemon osd.3 on smithi042 2024-04-02T14:58:52.338 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:52 smithi042 ceph-mon[29473]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:52.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:52 smithi169 ceph-mon[25801]: Deploying daemon osd.3 on smithi042 2024-04-02T14:58:52.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:52 smithi169 ceph-mon[25801]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:54.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:54 smithi042 ceph-mon[24805]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:54.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:54 smithi042 ceph-mon[29473]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:54.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:54 smithi169 ceph-mon[25801]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:56.456 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:56 smithi042 ceph-mon[24805]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:56.456 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:56 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:56.456 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:56 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:58:56.456 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:56 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:56.456 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:56 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:58:56.456 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:56 smithi042 ceph-mon[29473]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:56.456 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:56 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:56.457 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:56 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:58:56.457 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:56 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:56.457 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:56 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:58:56.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:56 smithi169 ceph-mon[25801]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:56.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:56 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:56.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:56 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:58:56.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:56 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:58:56.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:56 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:58:58.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:58 smithi169 ceph-mon[25801]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:58.501 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:58 smithi042 ceph-mon[24805]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:58.501 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:58 smithi042 ceph-mon[29473]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:58:58.532 INFO:teuthology.orchestra.run.smithi042.stdout:Created osd(s) 3 on host 'smithi042' 2024-04-02T14:58:59.044 DEBUG:teuthology.orchestra.run.smithi042:osd.3> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.3.service 2024-04-02T14:58:59.046 INFO:tasks.cephadm:Deploying osd.4 on smithi169 with /dev/vg_nvme/lv_4... 2024-04-02T14:58:59.046 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-02T14:58:59.635 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:59 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:59.635 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:58:59 smithi169 ceph-mon[25801]: from='osd.3 [v2:172.21.15.42:6826/1633099631,v1:172.21.15.42:6827/1633099631]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T14:58:59.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:59 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:59.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:58:59 smithi042 ceph-mon[24805]: from='osd.3 [v2:172.21.15.42:6826/1633099631,v1:172.21.15.42:6827/1633099631]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T14:58:59.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:59 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:58:59.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:58:59 smithi042 ceph-mon[29473]: from='osd.3 [v2:172.21.15.42:6826/1633099631,v1:172.21.15.42:6827/1633099631]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T14:59:00.657 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:00 smithi169 ceph-mon[25801]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:59:00.657 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:00 smithi169 ceph-mon[25801]: from='osd.3 [v2:172.21.15.42:6826/1633099631,v1:172.21.15.42:6827/1633099631]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T14:59:00.657 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:00 smithi169 ceph-mon[25801]: osdmap e24: 4 total, 3 up, 4 in 2024-04-02T14:59:00.657 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:00 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:59:00.657 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:00 smithi169 ceph-mon[25801]: from='osd.3 [v2:172.21.15.42:6826/1633099631,v1:172.21.15.42:6827/1633099631]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:59:00.657 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:00 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:00.677 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:00 smithi042 ceph-mon[24805]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:59:00.677 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:00 smithi042 ceph-mon[24805]: from='osd.3 [v2:172.21.15.42:6826/1633099631,v1:172.21.15.42:6827/1633099631]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T14:59:00.677 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:00 smithi042 ceph-mon[24805]: osdmap e24: 4 total, 3 up, 4 in 2024-04-02T14:59:00.677 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:00 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:59:00.677 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:00 smithi042 ceph-mon[24805]: from='osd.3 [v2:172.21.15.42:6826/1633099631,v1:172.21.15.42:6827/1633099631]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:59:00.677 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:00 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:00.678 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:00 smithi042 ceph-mon[29473]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:59:00.678 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:00 smithi042 ceph-mon[29473]: from='osd.3 [v2:172.21.15.42:6826/1633099631,v1:172.21.15.42:6827/1633099631]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T14:59:00.678 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:00 smithi042 ceph-mon[29473]: osdmap e24: 4 total, 3 up, 4 in 2024-04-02T14:59:00.678 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:00 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:59:00.678 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:00 smithi042 ceph-mon[29473]: from='osd.3 [v2:172.21.15.42:6826/1633099631,v1:172.21.15.42:6827/1633099631]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-04-02T14:59:00.678 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:00 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:00.678 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 14:59:00 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-3[41200]: 2024-04-02T14:59:00.551+0000 7fde90da6700 -1 osd.3 0 waiting for initial osdmap 2024-04-02T14:59:00.678 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 14:59:00 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-3[41200]: 2024-04-02T14:59:00.555+0000 7fde8bf3e700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T14:59:01.115 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-02T14:59:01.115 INFO:teuthology.orchestra.run.smithi169.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-02T14:59:01.115 INFO:teuthology.orchestra.run.smithi169.stderr: stderr: 10+0 records in 2024-04-02T14:59:01.115 INFO:teuthology.orchestra.run.smithi169.stderr:10+0 records out 2024-04-02T14:59:01.115 INFO:teuthology.orchestra.run.smithi169.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0131152 s, 800 MB/s 2024-04-02T14:59:01.115 INFO:teuthology.orchestra.run.smithi169.stderr: stderr: 2024-04-02T14:59:01.115 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping successful for: 2024-04-02T14:59:01.591 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:01 smithi042 ceph-mon[24805]: from='osd.3 [v2:172.21.15.42:6826/1633099631,v1:172.21.15.42:6827/1633099631]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-04-02T14:59:01.591 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:01 smithi042 ceph-mon[24805]: osdmap e25: 4 total, 3 up, 4 in 2024-04-02T14:59:01.591 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:01 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:59:01.591 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:01 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:59:01.591 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:01 smithi042 ceph-mon[29473]: from='osd.3 [v2:172.21.15.42:6826/1633099631,v1:172.21.15.42:6827/1633099631]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-04-02T14:59:01.591 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:01 smithi042 ceph-mon[29473]: osdmap e25: 4 total, 3 up, 4 in 2024-04-02T14:59:01.591 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:01 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:59:01.592 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:01 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:59:01.694 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:01 smithi169 ceph-mon[25801]: from='osd.3 [v2:172.21.15.42:6826/1633099631,v1:172.21.15.42:6827/1633099631]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-04-02T14:59:01.694 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:01 smithi169 ceph-mon[25801]: osdmap e25: 4 total, 3 up, 4 in 2024-04-02T14:59:01.694 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:01 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:59:01.694 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:01 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:59:01.717 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi169:vg_nvme/lv_4 2024-04-02T14:59:02.625 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:02 smithi169 ceph-mon[25801]: purged_snaps scrub starts 2024-04-02T14:59:02.625 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:02 smithi169 ceph-mon[25801]: purged_snaps scrub ok 2024-04-02T14:59:02.625 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:02 smithi169 ceph-mon[25801]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:59:02.626 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:02 smithi169 ceph-mon[25801]: osd.3 [v2:172.21.15.42:6826/1633099631,v1:172.21.15.42:6827/1633099631] boot 2024-04-02T14:59:02.626 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:02 smithi169 ceph-mon[25801]: osdmap e26: 4 total, 4 up, 4 in 2024-04-02T14:59:02.626 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:02 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:59:02.787 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:02 smithi042 ceph-mon[24805]: purged_snaps scrub starts 2024-04-02T14:59:02.787 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:02 smithi042 ceph-mon[24805]: purged_snaps scrub ok 2024-04-02T14:59:02.787 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:02 smithi042 ceph-mon[24805]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:59:02.787 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:02 smithi042 ceph-mon[24805]: osd.3 [v2:172.21.15.42:6826/1633099631,v1:172.21.15.42:6827/1633099631] boot 2024-04-02T14:59:02.787 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:02 smithi042 ceph-mon[24805]: osdmap e26: 4 total, 4 up, 4 in 2024-04-02T14:59:02.787 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:02 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:59:02.788 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:02 smithi042 ceph-mon[29473]: purged_snaps scrub starts 2024-04-02T14:59:02.788 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:02 smithi042 ceph-mon[29473]: purged_snaps scrub ok 2024-04-02T14:59:02.788 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:02 smithi042 ceph-mon[29473]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:59:02.788 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:02 smithi042 ceph-mon[29473]: osd.3 [v2:172.21.15.42:6826/1633099631,v1:172.21.15.42:6827/1633099631] boot 2024-04-02T14:59:02.788 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:02 smithi042 ceph-mon[29473]: osdmap e26: 4 total, 4 up, 4 in 2024-04-02T14:59:02.788 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:02 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:59:03.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[24805]: osdmap e27: 4 total, 4 up, 4 in 2024-04-02T14:59:03.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:03.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:03.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:03.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:03.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:03.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:03.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:03.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:59:03.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:03.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:59:03.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:03.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:59:03.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:59:03.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:03.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[29473]: osdmap e27: 4 total, 4 up, 4 in 2024-04-02T14:59:03.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:03.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:03.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:03.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:03.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:03.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:03.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:03.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:59:03.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:03.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:59:03.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:03.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:59:03.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:59:03.796 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:03.894 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:03 smithi169 ceph-mon[25801]: osdmap e27: 4 total, 4 up, 4 in 2024-04-02T14:59:03.894 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:03.894 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:03.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:03.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:03.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:03.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:03.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:03.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:59:03.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:03.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:59:03.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:03.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:59:03.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:59:03.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:04.843 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:04 smithi169 ceph-mon[25801]: Detected new or changed devices on smithi042 2024-04-02T14:59:04.843 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:04 smithi169 ceph-mon[25801]: Adjusting osd_memory_target on smithi042 to 4017M 2024-04-02T14:59:04.843 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:04 smithi169 ceph-mon[25801]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:04.843 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:04 smithi169 ceph-mon[25801]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:59:05.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:04 smithi042 ceph-mon[24805]: Detected new or changed devices on smithi042 2024-04-02T14:59:05.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:04 smithi042 ceph-mon[24805]: Adjusting osd_memory_target on smithi042 to 4017M 2024-04-02T14:59:05.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:04 smithi042 ceph-mon[24805]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:05.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:04 smithi042 ceph-mon[24805]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:59:05.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:04 smithi042 ceph-mon[29473]: Detected new or changed devices on smithi042 2024-04-02T14:59:05.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:04 smithi042 ceph-mon[29473]: Adjusting osd_memory_target on smithi042 to 4017M 2024-04-02T14:59:05.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:04 smithi042 ceph-mon[29473]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:05.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:04 smithi042 ceph-mon[29473]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:59:06.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:06 smithi169 ceph-mon[25801]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:06.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:06 smithi169 ceph-mon[25801]: from='client.? 172.21.15.169:0/3246434319' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "32958af8-ae8a-4abc-95d9-7c08ac3f758e"}]: dispatch 2024-04-02T14:59:06.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:06 smithi169 ceph-mon[25801]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "32958af8-ae8a-4abc-95d9-7c08ac3f758e"}]: dispatch 2024-04-02T14:59:06.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:06 smithi169 ceph-mon[25801]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "32958af8-ae8a-4abc-95d9-7c08ac3f758e"}]': finished 2024-04-02T14:59:06.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:06 smithi169 ceph-mon[25801]: osdmap e28: 5 total, 4 up, 5 in 2024-04-02T14:59:06.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:06 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:59:06.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:06 smithi169 ceph-mon[25801]: from='client.? 172.21.15.169:0/1362823071' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:59:07.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:06 smithi042 ceph-mon[24805]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:07.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:06 smithi042 ceph-mon[24805]: from='client.? 172.21.15.169:0/3246434319' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "32958af8-ae8a-4abc-95d9-7c08ac3f758e"}]: dispatch 2024-04-02T14:59:07.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:06 smithi042 ceph-mon[24805]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "32958af8-ae8a-4abc-95d9-7c08ac3f758e"}]: dispatch 2024-04-02T14:59:07.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:06 smithi042 ceph-mon[24805]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "32958af8-ae8a-4abc-95d9-7c08ac3f758e"}]': finished 2024-04-02T14:59:07.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:06 smithi042 ceph-mon[24805]: osdmap e28: 5 total, 4 up, 5 in 2024-04-02T14:59:07.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:06 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:59:07.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:06 smithi042 ceph-mon[24805]: from='client.? 172.21.15.169:0/1362823071' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:59:07.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:06 smithi042 ceph-mon[29473]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:07.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:06 smithi042 ceph-mon[29473]: from='client.? 172.21.15.169:0/3246434319' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "32958af8-ae8a-4abc-95d9-7c08ac3f758e"}]: dispatch 2024-04-02T14:59:07.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:06 smithi042 ceph-mon[29473]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "32958af8-ae8a-4abc-95d9-7c08ac3f758e"}]: dispatch 2024-04-02T14:59:07.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:06 smithi042 ceph-mon[29473]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "32958af8-ae8a-4abc-95d9-7c08ac3f758e"}]': finished 2024-04-02T14:59:07.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:06 smithi042 ceph-mon[29473]: osdmap e28: 5 total, 4 up, 5 in 2024-04-02T14:59:07.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:06 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:59:07.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:06 smithi042 ceph-mon[29473]: from='client.? 172.21.15.169:0/1362823071' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:59:08.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:08 smithi169 ceph-mon[25801]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:09.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:08 smithi042 ceph-mon[24805]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:09.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:08 smithi042 ceph-mon[29473]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:10.793 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:10 smithi169 ceph-mon[25801]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:11.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:10 smithi042 ceph-mon[24805]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:11.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:10 smithi042 ceph-mon[29473]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:12.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:12 smithi169 ceph-mon[25801]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:12.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:12 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T14:59:12.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:12 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:13.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:12 smithi042 ceph-mon[24805]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:13.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:12 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T14:59:13.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:12 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:13.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:12 smithi042 ceph-mon[29473]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:13.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:12 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T14:59:13.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:12 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:13.711 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:13 smithi169 ceph-mon[25801]: Deploying daemon osd.4 on smithi169 2024-04-02T14:59:14.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:13 smithi042 ceph-mon[24805]: Deploying daemon osd.4 on smithi169 2024-04-02T14:59:14.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:13 smithi042 ceph-mon[29473]: Deploying daemon osd.4 on smithi169 2024-04-02T14:59:14.922 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:14 smithi169 ceph-mon[25801]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:15.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:14 smithi042 ceph-mon[24805]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:15.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:14 smithi042 ceph-mon[29473]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:16.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:16 smithi169 ceph-mon[25801]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:16.846 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:16 smithi042 ceph-mon[24805]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:16.846 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:16 smithi042 ceph-mon[29473]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:17.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:17 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:17.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:17 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:59:17.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:17 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:17.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:17 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:59:17.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:17 smithi169 ceph-mon[25801]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:18.031 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:17 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:18.031 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:17 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:59:18.031 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:17 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:18.031 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:17 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:59:18.031 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:17 smithi042 ceph-mon[24805]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:18.032 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:17 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:18.032 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:17 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:59:18.032 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:17 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:18.032 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:17 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:59:18.032 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:17 smithi042 ceph-mon[29473]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:19.890 INFO:teuthology.orchestra.run.smithi169.stdout:Created osd(s) 4 on host 'smithi169' 2024-04-02T14:59:20.188 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:19 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:20.189 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:19 smithi169 ceph-mon[25801]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:20.189 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:19 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:20.189 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:19 smithi169 ceph-mon[25801]: from='osd.4 [v2:172.21.15.169:6800/2353291645,v1:172.21.15.169:6801/2353291645]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T14:59:20.189 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:19 smithi169 ceph-mon[25801]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T14:59:20.189 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:19 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:59:20.189 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:19 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:20.189 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:19 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:20.189 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:19 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:59:20.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:20.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[24805]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:20.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:20.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[24805]: from='osd.4 [v2:172.21.15.169:6800/2353291645,v1:172.21.15.169:6801/2353291645]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T14:59:20.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[24805]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T14:59:20.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:59:20.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:20.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:20.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:59:20.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:20.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[29473]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:20.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:20.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[29473]: from='osd.4 [v2:172.21.15.169:6800/2353291645,v1:172.21.15.169:6801/2353291645]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T14:59:20.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[29473]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T14:59:20.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:59:20.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:20.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:20.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:19 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:59:20.494 DEBUG:teuthology.orchestra.run.smithi169:osd.4> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.4.service 2024-04-02T14:59:20.496 INFO:tasks.cephadm:Deploying osd.5 on smithi169 with /dev/vg_nvme/lv_3... 2024-04-02T14:59:20.497 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-02T14:59:21.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:21 smithi042 ceph-mon[24805]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T14:59:21.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:21 smithi042 ceph-mon[24805]: osdmap e29: 5 total, 4 up, 5 in 2024-04-02T14:59:21.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:21 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:59:21.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:21 smithi042 ceph-mon[24805]: from='osd.4 [v2:172.21.15.169:6800/2353291645,v1:172.21.15.169:6801/2353291645]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T14:59:21.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:21 smithi042 ceph-mon[24805]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T14:59:21.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:21 smithi042 ceph-mon[29473]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T14:59:21.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:21 smithi042 ceph-mon[29473]: osdmap e29: 5 total, 4 up, 5 in 2024-04-02T14:59:21.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:21 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:59:21.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:21 smithi042 ceph-mon[29473]: from='osd.4 [v2:172.21.15.169:6800/2353291645,v1:172.21.15.169:6801/2353291645]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T14:59:21.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:21 smithi042 ceph-mon[29473]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T14:59:21.554 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:21 smithi169 ceph-mon[25801]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T14:59:21.554 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:21 smithi169 ceph-mon[25801]: osdmap e29: 5 total, 4 up, 5 in 2024-04-02T14:59:21.554 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:21 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:59:21.555 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:21 smithi169 ceph-mon[25801]: from='osd.4 [v2:172.21.15.169:6800/2353291645,v1:172.21.15.169:6801/2353291645]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T14:59:21.555 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:21 smithi169 ceph-mon[25801]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T14:59:21.555 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 14:59:21 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T14:59:21.286+0000 7f8a49284700 -1 osd.4 0 waiting for initial osdmap 2024-04-02T14:59:21.555 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 14:59:21 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T14:59:21.290+0000 7f8a45c1f700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T14:59:22.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:22 smithi169 ceph-mon[25801]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:22.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:22 smithi169 ceph-mon[25801]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2024-04-02T14:59:22.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:22 smithi169 ceph-mon[25801]: osdmap e30: 5 total, 4 up, 5 in 2024-04-02T14:59:22.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:22 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:59:22.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:22 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:59:22.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:22 smithi042 ceph-mon[24805]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:22.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:22 smithi042 ceph-mon[24805]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2024-04-02T14:59:22.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:22 smithi042 ceph-mon[24805]: osdmap e30: 5 total, 4 up, 5 in 2024-04-02T14:59:22.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:22 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:59:22.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:22 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:59:22.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:22 smithi042 ceph-mon[29473]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:59:22.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:22 smithi042 ceph-mon[29473]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2024-04-02T14:59:22.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:22 smithi042 ceph-mon[29473]: osdmap e30: 5 total, 4 up, 5 in 2024-04-02T14:59:22.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:22 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:59:22.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:22 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:59:23.272 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-02T14:59:23.272 INFO:teuthology.orchestra.run.smithi169.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-02T14:59:23.272 INFO:teuthology.orchestra.run.smithi169.stderr: stderr: 10+0 records in 2024-04-02T14:59:23.272 INFO:teuthology.orchestra.run.smithi169.stderr:10+0 records out 2024-04-02T14:59:23.272 INFO:teuthology.orchestra.run.smithi169.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0124183 s, 844 MB/s 2024-04-02T14:59:23.272 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping successful for: 2024-04-02T14:59:23.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:23 smithi169 ceph-mon[25801]: purged_snaps scrub starts 2024-04-02T14:59:23.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:23 smithi169 ceph-mon[25801]: purged_snaps scrub ok 2024-04-02T14:59:23.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:23 smithi169 ceph-mon[25801]: osd.4 [v2:172.21.15.169:6800/2353291645,v1:172.21.15.169:6801/2353291645] boot 2024-04-02T14:59:23.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:23 smithi169 ceph-mon[25801]: osdmap e31: 5 total, 5 up, 5 in 2024-04-02T14:59:23.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:23 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:59:23.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:23 smithi169 ceph-mon[25801]: osdmap e32: 5 total, 5 up, 5 in 2024-04-02T14:59:23.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:23 smithi042 ceph-mon[24805]: purged_snaps scrub starts 2024-04-02T14:59:23.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:23 smithi042 ceph-mon[24805]: purged_snaps scrub ok 2024-04-02T14:59:23.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:23 smithi042 ceph-mon[24805]: osd.4 [v2:172.21.15.169:6800/2353291645,v1:172.21.15.169:6801/2353291645] boot 2024-04-02T14:59:23.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:23 smithi042 ceph-mon[24805]: osdmap e31: 5 total, 5 up, 5 in 2024-04-02T14:59:23.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:23 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:59:23.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:23 smithi042 ceph-mon[24805]: osdmap e32: 5 total, 5 up, 5 in 2024-04-02T14:59:23.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:23 smithi042 ceph-mon[29473]: purged_snaps scrub starts 2024-04-02T14:59:23.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:23 smithi042 ceph-mon[29473]: purged_snaps scrub ok 2024-04-02T14:59:23.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:23 smithi042 ceph-mon[29473]: osd.4 [v2:172.21.15.169:6800/2353291645,v1:172.21.15.169:6801/2353291645] boot 2024-04-02T14:59:23.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:23 smithi042 ceph-mon[29473]: osdmap e31: 5 total, 5 up, 5 in 2024-04-02T14:59:23.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:23 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:59:23.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:23 smithi042 ceph-mon[29473]: osdmap e32: 5 total, 5 up, 5 in 2024-04-02T14:59:23.987 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi169:vg_nvme/lv_3 2024-04-02T14:59:24.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:24 smithi042 ceph-mon[24805]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:24.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:24 smithi042 ceph-mon[24805]: osdmap e33: 5 total, 5 up, 5 in 2024-04-02T14:59:24.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:24 smithi042 ceph-mon[29473]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:24.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:24 smithi042 ceph-mon[29473]: osdmap e33: 5 total, 5 up, 5 in 2024-04-02T14:59:24.599 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:24 smithi169 ceph-mon[25801]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:24.599 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:24 smithi169 ceph-mon[25801]: osdmap e33: 5 total, 5 up, 5 in 2024-04-02T14:59:25.807 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:25 smithi169 ceph-mon[25801]: Detected new or changed devices on smithi169 2024-04-02T14:59:25.807 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:25 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:25.807 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:25 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:25.807 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:25 smithi169 ceph-mon[25801]: Adjusting osd_memory_target on smithi169 to 17088M 2024-04-02T14:59:25.808 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:25 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:25.808 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:25 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:25.808 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:25 smithi169 ceph-mon[25801]: osdmap e34: 5 total, 5 up, 5 in 2024-04-02T14:59:26.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:25 smithi042 ceph-mon[24805]: Detected new or changed devices on smithi169 2024-04-02T14:59:26.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:25 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:26.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:25 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:26.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:25 smithi042 ceph-mon[24805]: Adjusting osd_memory_target on smithi169 to 17088M 2024-04-02T14:59:26.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:25 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:26.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:25 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:26.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:25 smithi042 ceph-mon[24805]: osdmap e34: 5 total, 5 up, 5 in 2024-04-02T14:59:26.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:25 smithi042 ceph-mon[29473]: Detected new or changed devices on smithi169 2024-04-02T14:59:26.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:25 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:26.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:25 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:26.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:25 smithi042 ceph-mon[29473]: Adjusting osd_memory_target on smithi169 to 17088M 2024-04-02T14:59:26.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:25 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:26.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:25 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:26.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:25 smithi042 ceph-mon[29473]: osdmap e34: 5 total, 5 up, 5 in 2024-04-02T14:59:26.843 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:26 smithi169 ceph-mon[25801]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:26.844 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:26 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:59:26.844 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:26 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:59:26.844 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:26 smithi169 ceph-mon[25801]: osdmap e35: 5 total, 5 up, 5 in 2024-04-02T14:59:26.844 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:26 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:27.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:26 smithi042 ceph-mon[24805]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:27.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:26 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:59:27.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:26 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:59:27.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:26 smithi042 ceph-mon[24805]: osdmap e35: 5 total, 5 up, 5 in 2024-04-02T14:59:27.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:26 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:27.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:26 smithi042 ceph-mon[29473]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:27.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:26 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:59:27.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:26 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:59:27.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:26 smithi042 ceph-mon[29473]: osdmap e35: 5 total, 5 up, 5 in 2024-04-02T14:59:27.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:26 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:27.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:27 smithi042 ceph-mon[24805]: from='client.24233 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:59:27.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:27 smithi042 ceph-mon[29473]: from='client.24233 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:59:27.900 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:27 smithi169 ceph-mon[25801]: from='client.24233 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:59:28.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:28 smithi169 ceph-mon[25801]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:28.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:28 smithi169 ceph-mon[25801]: from='client.? 172.21.15.169:0/1803605250' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "46d22890-4110-43a7-9c0d-7406495a9924"}]: dispatch 2024-04-02T14:59:28.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:28 smithi169 ceph-mon[25801]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "46d22890-4110-43a7-9c0d-7406495a9924"}]: dispatch 2024-04-02T14:59:28.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:28 smithi169 ceph-mon[25801]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "46d22890-4110-43a7-9c0d-7406495a9924"}]': finished 2024-04-02T14:59:28.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:28 smithi169 ceph-mon[25801]: osdmap e36: 6 total, 5 up, 6 in 2024-04-02T14:59:28.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:28 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:59:29.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:28 smithi042 ceph-mon[24805]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:29.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:28 smithi042 ceph-mon[24805]: from='client.? 172.21.15.169:0/1803605250' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "46d22890-4110-43a7-9c0d-7406495a9924"}]: dispatch 2024-04-02T14:59:29.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:28 smithi042 ceph-mon[24805]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "46d22890-4110-43a7-9c0d-7406495a9924"}]: dispatch 2024-04-02T14:59:29.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:28 smithi042 ceph-mon[24805]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "46d22890-4110-43a7-9c0d-7406495a9924"}]': finished 2024-04-02T14:59:29.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:28 smithi042 ceph-mon[24805]: osdmap e36: 6 total, 5 up, 6 in 2024-04-02T14:59:29.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:28 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:59:29.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:28 smithi042 ceph-mon[29473]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:29.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:28 smithi042 ceph-mon[29473]: from='client.? 172.21.15.169:0/1803605250' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "46d22890-4110-43a7-9c0d-7406495a9924"}]: dispatch 2024-04-02T14:59:29.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:28 smithi042 ceph-mon[29473]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "46d22890-4110-43a7-9c0d-7406495a9924"}]: dispatch 2024-04-02T14:59:29.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:28 smithi042 ceph-mon[29473]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "46d22890-4110-43a7-9c0d-7406495a9924"}]': finished 2024-04-02T14:59:29.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:28 smithi042 ceph-mon[29473]: osdmap e36: 6 total, 5 up, 6 in 2024-04-02T14:59:29.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:28 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:59:29.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:29 smithi169 ceph-mon[25801]: from='client.? 172.21.15.169:0/2543556832' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:59:30.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:29 smithi042 ceph-mon[24805]: from='client.? 172.21.15.169:0/2543556832' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:59:30.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:29 smithi042 ceph-mon[29473]: from='client.? 172.21.15.169:0/2543556832' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:59:30.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:30 smithi169 ceph-mon[25801]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:31.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:30 smithi042 ceph-mon[24805]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:31.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:30 smithi042 ceph-mon[29473]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:31.855 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:31 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:59:31.855 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:31 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T14:59:32.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:59:32.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T14:59:32.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:31 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:59:32.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:31 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T14:59:32.894 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:32 smithi169 ceph-mon[25801]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T14:59:33.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:32 smithi042 ceph-mon[24805]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T14:59:33.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:32 smithi042 ceph-mon[29473]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T14:59:34.915 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:34 smithi169 ceph-mon[25801]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T14:59:35.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:34 smithi042 ceph-mon[24805]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T14:59:35.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:34 smithi042 ceph-mon[29473]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T14:59:35.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:35 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T14:59:35.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:35 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:35.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:35 smithi169 ceph-mon[25801]: Deploying daemon osd.5 on smithi169 2024-04-02T14:59:36.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:35 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T14:59:36.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:35 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:36.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:35 smithi042 ceph-mon[29473]: Deploying daemon osd.5 on smithi169 2024-04-02T14:59:36.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:35 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T14:59:36.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:35 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:36.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:35 smithi042 ceph-mon[24805]: Deploying daemon osd.5 on smithi169 2024-04-02T14:59:36.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:36 smithi169 ceph-mon[25801]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-04-02T14:59:37.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:36 smithi042 ceph-mon[24805]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-04-02T14:59:37.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:36 smithi042 ceph-mon[29473]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-04-02T14:59:38.938 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:38 smithi169 ceph-mon[25801]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T14:59:39.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:38 smithi042 ceph-mon[24805]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T14:59:39.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:38 smithi042 ceph-mon[29473]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T14:59:40.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:40 smithi169 ceph-mon[25801]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-02T14:59:40.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:40 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:40.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:40 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:59:40.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:40 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:40.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:40 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:59:40.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:40 smithi042 ceph-mon[24805]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-02T14:59:40.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:40 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:40.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:40 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:59:40.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:40 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:40.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:40 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:59:40.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:40 smithi042 ceph-mon[29473]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-02T14:59:40.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:40 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:40.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:40 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:59:40.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:40 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:40.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:40 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:59:42.437 INFO:teuthology.orchestra.run.smithi169.stdout:Created osd(s) 5 on host 'smithi169' 2024-04-02T14:59:42.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:42 smithi169 ceph-mon[25801]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-02T14:59:42.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:42 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:42.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:42 smithi042 ceph-mon[24805]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-02T14:59:42.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:42 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:42.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:42 smithi042 ceph-mon[29473]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-02T14:59:42.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:42 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:43.049 DEBUG:teuthology.orchestra.run.smithi169:osd.5> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.5.service 2024-04-02T14:59:43.052 INFO:tasks.cephadm:Deploying osd.6 on smithi169 with /dev/vg_nvme/lv_2... 2024-04-02T14:59:43.052 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-02T14:59:43.545 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:43 smithi169 ceph-mon[25801]: from='osd.5 [v2:172.21.15.169:6808/1262879371,v1:172.21.15.169:6809/1262879371]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T14:59:43.545 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:43 smithi169 ceph-mon[25801]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T14:59:43.545 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:43 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:43.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:43 smithi042 ceph-mon[24805]: from='osd.5 [v2:172.21.15.169:6808/1262879371,v1:172.21.15.169:6809/1262879371]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T14:59:43.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:43 smithi042 ceph-mon[24805]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T14:59:43.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:43 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:43.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:43 smithi042 ceph-mon[29473]: from='osd.5 [v2:172.21.15.169:6808/1262879371,v1:172.21.15.169:6809/1262879371]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T14:59:43.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:43 smithi042 ceph-mon[29473]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T14:59:43.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:43 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:44.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:44 smithi169 ceph-mon[25801]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:44.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:44 smithi169 ceph-mon[25801]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T14:59:44.495 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:44 smithi169 ceph-mon[25801]: osdmap e37: 6 total, 5 up, 6 in 2024-04-02T14:59:44.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:44 smithi042 ceph-mon[24805]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:44.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:44 smithi042 ceph-mon[24805]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T14:59:44.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:44 smithi042 ceph-mon[24805]: osdmap e37: 6 total, 5 up, 6 in 2024-04-02T14:59:44.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:44 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:59:44.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:44 smithi042 ceph-mon[24805]: from='osd.5 [v2:172.21.15.169:6808/1262879371,v1:172.21.15.169:6809/1262879371]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T14:59:44.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:44 smithi042 ceph-mon[24805]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T14:59:44.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:44 smithi042 ceph-mon[29473]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:44.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:44 smithi042 ceph-mon[29473]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T14:59:44.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:44 smithi042 ceph-mon[29473]: osdmap e37: 6 total, 5 up, 6 in 2024-04-02T14:59:44.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:44 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:59:44.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:44 smithi042 ceph-mon[29473]: from='osd.5 [v2:172.21.15.169:6808/1262879371,v1:172.21.15.169:6809/1262879371]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T14:59:44.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:44 smithi042 ceph-mon[29473]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T14:59:44.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:44 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:59:44.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:44 smithi169 ceph-mon[25801]: from='osd.5 [v2:172.21.15.169:6808/1262879371,v1:172.21.15.169:6809/1262879371]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T14:59:44.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:44 smithi169 ceph-mon[25801]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T14:59:44.978 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 14:59:44 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T14:59:44.497+0000 7fa0c8c65700 -1 osd.5 0 waiting for initial osdmap 2024-04-02T14:59:44.979 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 14:59:44 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T14:59:44.504+0000 7fa0c3dfd700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T14:59:45.610 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-02T14:59:45.610 INFO:teuthology.orchestra.run.smithi169.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-02T14:59:45.610 INFO:teuthology.orchestra.run.smithi169.stderr: stderr: 10+0 records in 2024-04-02T14:59:45.610 INFO:teuthology.orchestra.run.smithi169.stderr:10+0 records out 2024-04-02T14:59:45.610 INFO:teuthology.orchestra.run.smithi169.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0142034 s, 738 MB/s 2024-04-02T14:59:45.610 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping successful for: 2024-04-02T14:59:45.610 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:45 smithi169 ceph-mon[25801]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2024-04-02T14:59:45.611 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:45 smithi169 ceph-mon[25801]: osdmap e38: 6 total, 5 up, 6 in 2024-04-02T14:59:45.611 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:45 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:59:45.611 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:45 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:59:45.611 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:45 smithi169 ceph-mon[25801]: osd.5 [v2:172.21.15.169:6808/1262879371,v1:172.21.15.169:6809/1262879371] boot 2024-04-02T14:59:45.611 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:45 smithi169 ceph-mon[25801]: osdmap e39: 6 total, 6 up, 6 in 2024-04-02T14:59:45.611 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:45 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:59:45.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:45 smithi042 ceph-mon[24805]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2024-04-02T14:59:45.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:45 smithi042 ceph-mon[24805]: osdmap e38: 6 total, 5 up, 6 in 2024-04-02T14:59:45.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:45 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:59:45.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:45 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:59:45.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:45 smithi042 ceph-mon[24805]: osd.5 [v2:172.21.15.169:6808/1262879371,v1:172.21.15.169:6809/1262879371] boot 2024-04-02T14:59:45.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:45 smithi042 ceph-mon[24805]: osdmap e39: 6 total, 6 up, 6 in 2024-04-02T14:59:45.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:45 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:59:45.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:45 smithi042 ceph-mon[29473]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2024-04-02T14:59:45.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:45 smithi042 ceph-mon[29473]: osdmap e38: 6 total, 5 up, 6 in 2024-04-02T14:59:45.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:45 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:59:45.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:45 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:59:45.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:45 smithi042 ceph-mon[29473]: osd.5 [v2:172.21.15.169:6808/1262879371,v1:172.21.15.169:6809/1262879371] boot 2024-04-02T14:59:45.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:45 smithi042 ceph-mon[29473]: osdmap e39: 6 total, 6 up, 6 in 2024-04-02T14:59:45.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:45 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:59:46.313 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi169:vg_nvme/lv_2 2024-04-02T14:59:46.625 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:46 smithi169 ceph-mon[25801]: purged_snaps scrub starts 2024-04-02T14:59:46.625 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:46 smithi169 ceph-mon[25801]: purged_snaps scrub ok 2024-04-02T14:59:46.625 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:46 smithi169 ceph-mon[25801]: pgmap v125: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:46.625 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:46 smithi169 ceph-mon[25801]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-02T14:59:46.625 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:46 smithi169 ceph-mon[25801]: osdmap e40: 6 total, 6 up, 6 in 2024-04-02T14:59:46.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:46 smithi042 ceph-mon[24805]: purged_snaps scrub starts 2024-04-02T14:59:46.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:46 smithi042 ceph-mon[24805]: purged_snaps scrub ok 2024-04-02T14:59:46.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:46 smithi042 ceph-mon[24805]: pgmap v125: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:46.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:46 smithi042 ceph-mon[24805]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-02T14:59:46.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:46 smithi042 ceph-mon[24805]: osdmap e40: 6 total, 6 up, 6 in 2024-04-02T14:59:46.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:46 smithi042 ceph-mon[29473]: purged_snaps scrub starts 2024-04-02T14:59:46.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:46 smithi042 ceph-mon[29473]: purged_snaps scrub ok 2024-04-02T14:59:46.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:46 smithi042 ceph-mon[29473]: pgmap v125: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:59:46.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:46 smithi042 ceph-mon[29473]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-02T14:59:46.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:46 smithi042 ceph-mon[29473]: osdmap e40: 6 total, 6 up, 6 in 2024-04-02T14:59:48.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:48 smithi169 ceph-mon[25801]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:59:48.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:48 smithi169 ceph-mon[25801]: osdmap e41: 6 total, 6 up, 6 in 2024-04-02T14:59:48.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:48 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:48.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:48 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:48.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:48 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:48.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:48 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:48.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:48 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:48.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:48 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:59:48.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:48 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:48.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:48 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:59:48.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:48 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:48.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[24805]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:59:48.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[24805]: osdmap e41: 6 total, 6 up, 6 in 2024-04-02T14:59:48.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:48.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:48.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:48.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:48.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:48.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:59:48.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:48.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:59:48.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:48.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[29473]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:59:48.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[29473]: osdmap e41: 6 total, 6 up, 6 in 2024-04-02T14:59:48.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:48.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:48.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:59:48.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:48.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:48.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:59:48.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:48.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:59:48.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:48 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T14:59:49.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:49 smithi169 ceph-mon[25801]: Detected new or changed devices on smithi169 2024-04-02T14:59:49.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:49 smithi169 ceph-mon[25801]: Adjusting osd_memory_target on smithi169 to 8544M 2024-04-02T14:59:49.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:49 smithi169 ceph-mon[25801]: osdmap e42: 6 total, 6 up, 6 in 2024-04-02T14:59:49.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:49 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:59:49.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:49 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:59:49.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:49 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:49.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:49 smithi042 ceph-mon[24805]: Detected new or changed devices on smithi169 2024-04-02T14:59:49.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:49 smithi042 ceph-mon[24805]: Adjusting osd_memory_target on smithi169 to 8544M 2024-04-02T14:59:49.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:49 smithi042 ceph-mon[24805]: osdmap e42: 6 total, 6 up, 6 in 2024-04-02T14:59:49.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:49 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:59:49.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:49 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:59:49.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:49 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:49.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:49 smithi042 ceph-mon[29473]: Detected new or changed devices on smithi169 2024-04-02T14:59:49.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:49 smithi042 ceph-mon[29473]: Adjusting osd_memory_target on smithi169 to 8544M 2024-04-02T14:59:49.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:49 smithi042 ceph-mon[29473]: osdmap e42: 6 total, 6 up, 6 in 2024-04-02T14:59:49.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:49 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:59:49.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:49 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:59:49.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:49 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:50.472 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:50 smithi169 ceph-mon[25801]: from='client.24260 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:59:50.473 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:50 smithi169 ceph-mon[25801]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:59:50.473 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:50 smithi169 ceph-mon[25801]: osdmap e43: 6 total, 6 up, 6 in 2024-04-02T14:59:50.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:50 smithi042 ceph-mon[24805]: from='client.24260 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:59:50.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:50 smithi042 ceph-mon[24805]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:59:50.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:50 smithi042 ceph-mon[24805]: osdmap e43: 6 total, 6 up, 6 in 2024-04-02T14:59:50.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:50 smithi042 ceph-mon[29473]: from='client.24260 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:59:50.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:50 smithi042 ceph-mon[29473]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:59:50.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:50 smithi042 ceph-mon[29473]: osdmap e43: 6 total, 6 up, 6 in 2024-04-02T14:59:51.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:51 smithi042 ceph-mon[24805]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-02T14:59:51.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:51 smithi042 ceph-mon[24805]: Cluster is now healthy 2024-04-02T14:59:51.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:51 smithi042 ceph-mon[29473]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-02T14:59:51.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:51 smithi042 ceph-mon[29473]: Cluster is now healthy 2024-04-02T14:59:51.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:51 smithi169 ceph-mon[25801]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-02T14:59:51.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:51 smithi169 ceph-mon[25801]: Cluster is now healthy 2024-04-02T14:59:52.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:52 smithi042 ceph-mon[24805]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-02T14:59:52.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:52 smithi042 ceph-mon[24805]: from='client.? 172.21.15.169:0/3056935036' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aca7c09b-7ae9-45b2-91d2-48413a58886e"}]: dispatch 2024-04-02T14:59:52.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:52 smithi042 ceph-mon[24805]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aca7c09b-7ae9-45b2-91d2-48413a58886e"}]: dispatch 2024-04-02T14:59:52.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:52 smithi042 ceph-mon[24805]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aca7c09b-7ae9-45b2-91d2-48413a58886e"}]': finished 2024-04-02T14:59:52.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:52 smithi042 ceph-mon[24805]: osdmap e44: 7 total, 6 up, 7 in 2024-04-02T14:59:52.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:52 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T14:59:52.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:52 smithi042 ceph-mon[24805]: from='client.? 172.21.15.169:0/3902728431' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:59:52.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:52 smithi042 ceph-mon[29473]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-02T14:59:52.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:52 smithi042 ceph-mon[29473]: from='client.? 172.21.15.169:0/3056935036' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aca7c09b-7ae9-45b2-91d2-48413a58886e"}]: dispatch 2024-04-02T14:59:52.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:52 smithi042 ceph-mon[29473]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aca7c09b-7ae9-45b2-91d2-48413a58886e"}]: dispatch 2024-04-02T14:59:52.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:52 smithi042 ceph-mon[29473]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aca7c09b-7ae9-45b2-91d2-48413a58886e"}]': finished 2024-04-02T14:59:52.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:52 smithi042 ceph-mon[29473]: osdmap e44: 7 total, 6 up, 7 in 2024-04-02T14:59:52.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:52 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T14:59:52.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:52 smithi042 ceph-mon[29473]: from='client.? 172.21.15.169:0/3902728431' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:59:52.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:52 smithi169 ceph-mon[25801]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-02T14:59:52.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:52 smithi169 ceph-mon[25801]: from='client.? 172.21.15.169:0/3056935036' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aca7c09b-7ae9-45b2-91d2-48413a58886e"}]: dispatch 2024-04-02T14:59:52.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:52 smithi169 ceph-mon[25801]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aca7c09b-7ae9-45b2-91d2-48413a58886e"}]: dispatch 2024-04-02T14:59:52.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:52 smithi169 ceph-mon[25801]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aca7c09b-7ae9-45b2-91d2-48413a58886e"}]': finished 2024-04-02T14:59:52.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:52 smithi169 ceph-mon[25801]: osdmap e44: 7 total, 6 up, 7 in 2024-04-02T14:59:52.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:52 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T14:59:52.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:52 smithi169 ceph-mon[25801]: from='client.? 172.21.15.169:0/3902728431' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:59:54.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:54 smithi169 ceph-mon[25801]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T14:59:54.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:54 smithi042 ceph-mon[24805]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T14:59:54.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:54 smithi042 ceph-mon[29473]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T14:59:56.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:56 smithi042 ceph-mon[24805]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-02T14:59:56.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:56 smithi042 ceph-mon[29473]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-02T14:59:56.544 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:56 smithi169 ceph-mon[25801]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-02T14:59:58.457 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:58 smithi169 ceph-mon[25801]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T14:59:58.457 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:58 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T14:59:58.457 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:58 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:58.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:58 smithi042 ceph-mon[24805]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T14:59:58.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:58 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T14:59:58.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:58 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:58.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:58 smithi042 ceph-mon[29473]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T14:59:58.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:58 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T14:59:58.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:58 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:59:59.349 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 14:59:59 smithi169 ceph-mon[25801]: Deploying daemon osd.6 on smithi169 2024-04-02T14:59:59.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 14:59:59 smithi042 ceph-mon[24805]: Deploying daemon osd.6 on smithi169 2024-04-02T14:59:59.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 14:59:59 smithi042 ceph-mon[29473]: Deploying daemon osd.6 on smithi169 2024-04-02T15:00:00.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:00 smithi042 ceph-mon[24805]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T15:00:00.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:00 smithi042 ceph-mon[24805]: overall HEALTH_OK 2024-04-02T15:00:00.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:00 smithi042 ceph-mon[29473]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T15:00:00.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:00 smithi042 ceph-mon[29473]: overall HEALTH_OK 2024-04-02T15:00:00.614 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:00 smithi169 ceph-mon[25801]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T15:00:00.615 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:00 smithi169 ceph-mon[25801]: overall HEALTH_OK 2024-04-02T15:00:02.434 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:02 smithi169 ceph-mon[25801]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:00:02.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:02 smithi042 ceph-mon[24805]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:00:02.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:02 smithi042 ceph-mon[29473]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:00:03.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:03.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:03.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:03.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:03 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:03.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:03 smithi169 ceph-mon[25801]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:00:04.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:04.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:04.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:04.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:03 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:04.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:03 smithi042 ceph-mon[29473]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:00:04.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:04.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:04.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:04.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:03 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:04.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:03 smithi042 ceph-mon[24805]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:00:05.743 INFO:teuthology.orchestra.run.smithi169.stdout:Created osd(s) 6 on host 'smithi169' 2024-04-02T15:00:06.365 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:06 smithi169 ceph-mon[25801]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:00:06.365 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:06 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:06.366 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:06 smithi169 ceph-mon[25801]: from='osd.6 [v2:172.21.15.169:6816/3799859416,v1:172.21.15.169:6817/3799859416]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T15:00:06.366 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:06 smithi169 ceph-mon[25801]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T15:00:06.387 DEBUG:teuthology.orchestra.run.smithi169:osd.6> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.6.service 2024-04-02T15:00:06.389 INFO:tasks.cephadm:Deploying osd.7 on smithi169 with /dev/vg_nvme/lv_1... 2024-04-02T15:00:06.389 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-02T15:00:06.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:06 smithi042 ceph-mon[29473]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:00:06.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:06 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:06.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:06 smithi042 ceph-mon[29473]: from='osd.6 [v2:172.21.15.169:6816/3799859416,v1:172.21.15.169:6817/3799859416]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T15:00:06.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:06 smithi042 ceph-mon[29473]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T15:00:06.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:06 smithi042 ceph-mon[24805]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:00:06.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:06 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:06.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:06 smithi042 ceph-mon[24805]: from='osd.6 [v2:172.21.15.169:6816/3799859416,v1:172.21.15.169:6817/3799859416]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T15:00:06.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:06 smithi042 ceph-mon[24805]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T15:00:07.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:07 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:07.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:07 smithi169 ceph-mon[25801]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-02T15:00:07.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:07 smithi169 ceph-mon[25801]: osdmap e45: 7 total, 6 up, 7 in 2024-04-02T15:00:07.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:07 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:00:07.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:07 smithi169 ceph-mon[25801]: from='osd.6 [v2:172.21.15.169:6816/3799859416,v1:172.21.15.169:6817/3799859416]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T15:00:07.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:07 smithi169 ceph-mon[25801]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T15:00:07.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:07 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:07.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:07 smithi042 ceph-mon[29473]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-02T15:00:07.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:07 smithi042 ceph-mon[29473]: osdmap e45: 7 total, 6 up, 7 in 2024-04-02T15:00:07.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:07 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:00:07.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:07 smithi042 ceph-mon[29473]: from='osd.6 [v2:172.21.15.169:6816/3799859416,v1:172.21.15.169:6817/3799859416]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T15:00:07.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:07 smithi042 ceph-mon[29473]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T15:00:07.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:07 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:07.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:07 smithi042 ceph-mon[24805]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-02T15:00:07.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:07 smithi042 ceph-mon[24805]: osdmap e45: 7 total, 6 up, 7 in 2024-04-02T15:00:07.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:07 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:00:07.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:07 smithi042 ceph-mon[24805]: from='osd.6 [v2:172.21.15.169:6816/3799859416,v1:172.21.15.169:6817/3799859416]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T15:00:07.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:07 smithi042 ceph-mon[24805]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T15:00:08.067 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:00:07 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:00:07.760+0000 7ff6d14f8700 -1 osd.6 0 waiting for initial osdmap 2024-04-02T15:00:08.067 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:00:07 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:00:07.769+0000 7ff6cce91700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T15:00:08.587 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:08 smithi169 ceph-mon[25801]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:00:08.587 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:08 smithi169 ceph-mon[25801]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2024-04-02T15:00:08.587 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:08 smithi169 ceph-mon[25801]: osdmap e46: 7 total, 6 up, 7 in 2024-04-02T15:00:08.587 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:08 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:00:08.587 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:08 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:00:08.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:08 smithi042 ceph-mon[24805]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:00:08.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:08 smithi042 ceph-mon[24805]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2024-04-02T15:00:08.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:08 smithi042 ceph-mon[24805]: osdmap e46: 7 total, 6 up, 7 in 2024-04-02T15:00:08.796 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:08 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:00:08.797 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:08 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:00:08.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:08 smithi042 ceph-mon[29473]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:00:08.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:08 smithi042 ceph-mon[29473]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2024-04-02T15:00:08.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:08 smithi042 ceph-mon[29473]: osdmap e46: 7 total, 6 up, 7 in 2024-04-02T15:00:08.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:08 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:00:08.799 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:08 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:00:09.347 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-02T15:00:09.347 INFO:teuthology.orchestra.run.smithi169.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-02T15:00:09.347 INFO:teuthology.orchestra.run.smithi169.stderr: stderr: 10+0 records in 2024-04-02T15:00:09.347 INFO:teuthology.orchestra.run.smithi169.stderr:10+0 records out 2024-04-02T15:00:09.347 INFO:teuthology.orchestra.run.smithi169.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0138429 s, 757 MB/s 2024-04-02T15:00:09.347 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping successful for: 2024-04-02T15:00:10.011 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:09 smithi169 ceph-mon[25801]: purged_snaps scrub starts 2024-04-02T15:00:10.011 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:09 smithi169 ceph-mon[25801]: purged_snaps scrub ok 2024-04-02T15:00:10.011 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:09 smithi169 ceph-mon[25801]: osd.6 [v2:172.21.15.169:6816/3799859416,v1:172.21.15.169:6817/3799859416] boot 2024-04-02T15:00:10.011 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:09 smithi169 ceph-mon[25801]: osdmap e47: 7 total, 7 up, 7 in 2024-04-02T15:00:10.011 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:09 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:00:10.011 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:09 smithi169 ceph-mon[25801]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:00:10.013 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi169:vg_nvme/lv_1 2024-04-02T15:00:10.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:09 smithi042 ceph-mon[29473]: purged_snaps scrub starts 2024-04-02T15:00:10.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:09 smithi042 ceph-mon[29473]: purged_snaps scrub ok 2024-04-02T15:00:10.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:09 smithi042 ceph-mon[29473]: osd.6 [v2:172.21.15.169:6816/3799859416,v1:172.21.15.169:6817/3799859416] boot 2024-04-02T15:00:10.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:09 smithi042 ceph-mon[29473]: osdmap e47: 7 total, 7 up, 7 in 2024-04-02T15:00:10.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:09 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:00:10.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:09 smithi042 ceph-mon[29473]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:00:10.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:09 smithi042 ceph-mon[24805]: purged_snaps scrub starts 2024-04-02T15:00:10.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:09 smithi042 ceph-mon[24805]: purged_snaps scrub ok 2024-04-02T15:00:10.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:09 smithi042 ceph-mon[24805]: osd.6 [v2:172.21.15.169:6816/3799859416,v1:172.21.15.169:6817/3799859416] boot 2024-04-02T15:00:10.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:09 smithi042 ceph-mon[24805]: osdmap e47: 7 total, 7 up, 7 in 2024-04-02T15:00:10.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:09 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:00:10.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:09 smithi042 ceph-mon[24805]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:00:11.020 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:10 smithi169 ceph-mon[25801]: osdmap e48: 7 total, 7 up, 7 in 2024-04-02T15:00:11.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:10 smithi042 ceph-mon[24805]: osdmap e48: 7 total, 7 up, 7 in 2024-04-02T15:00:11.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:10 smithi042 ceph-mon[29473]: osdmap e48: 7 total, 7 up, 7 in 2024-04-02T15:00:12.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[24805]: osdmap e49: 7 total, 7 up, 7 in 2024-04-02T15:00:12.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[24805]: Detected new or changed devices on smithi169 2024-04-02T15:00:12.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:12.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:12.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:12.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:12.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[24805]: Adjusting osd_memory_target on smithi169 to 5696M 2024-04-02T15:00:12.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:12.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:12.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:12.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:12.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:12.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:12.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[24805]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:12.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[29473]: osdmap e49: 7 total, 7 up, 7 in 2024-04-02T15:00:12.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[29473]: Detected new or changed devices on smithi169 2024-04-02T15:00:12.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:12.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:12.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:12.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:12.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[29473]: Adjusting osd_memory_target on smithi169 to 5696M 2024-04-02T15:00:12.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:12.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:12.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:12.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:12.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:12.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:12.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:11 smithi042 ceph-mon[29473]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:12.112 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:11 smithi169 ceph-mon[25801]: osdmap e49: 7 total, 7 up, 7 in 2024-04-02T15:00:12.112 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:11 smithi169 ceph-mon[25801]: Detected new or changed devices on smithi169 2024-04-02T15:00:12.112 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:12.112 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:12.112 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:12.112 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:12.112 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:11 smithi169 ceph-mon[25801]: Adjusting osd_memory_target on smithi169 to 5696M 2024-04-02T15:00:12.112 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:12.113 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:12.113 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:12.113 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:12.113 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:12.113 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:11 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:12.113 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:11 smithi169 ceph-mon[25801]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:13.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:12 smithi042 ceph-mon[24805]: osdmap e50: 7 total, 7 up, 7 in 2024-04-02T15:00:13.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:12 smithi042 ceph-mon[24805]: from='client.24287 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:00:13.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:12 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:00:13.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:12 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:00:13.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:12 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:13.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:12 smithi042 ceph-mon[29473]: osdmap e50: 7 total, 7 up, 7 in 2024-04-02T15:00:13.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:12 smithi042 ceph-mon[29473]: from='client.24287 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:00:13.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:12 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:00:13.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:12 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:00:13.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:12 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:13.115 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:12 smithi169 ceph-mon[25801]: osdmap e50: 7 total, 7 up, 7 in 2024-04-02T15:00:13.115 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:12 smithi169 ceph-mon[25801]: from='client.24287 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:00:13.115 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:12 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:00:13.115 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:12 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:00:13.116 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:12 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:14.005 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:13 smithi169 ceph-mon[25801]: osdmap e51: 7 total, 7 up, 7 in 2024-04-02T15:00:14.006 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:13 smithi169 ceph-mon[25801]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:14.293 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:13 smithi042 ceph-mon[29473]: osdmap e51: 7 total, 7 up, 7 in 2024-04-02T15:00:14.293 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:13 smithi042 ceph-mon[29473]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:14.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:13 smithi042 ceph-mon[24805]: osdmap e51: 7 total, 7 up, 7 in 2024-04-02T15:00:14.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:13 smithi042 ceph-mon[24805]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:15.228 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:14 smithi169 ceph-mon[25801]: from='client.? 172.21.15.169:0/1641578283' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "661346d4-14d3-446f-93cc-8fc8f6fd9470"}]: dispatch 2024-04-02T15:00:15.228 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:14 smithi169 ceph-mon[25801]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "661346d4-14d3-446f-93cc-8fc8f6fd9470"}]: dispatch 2024-04-02T15:00:15.228 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:14 smithi169 ceph-mon[25801]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "661346d4-14d3-446f-93cc-8fc8f6fd9470"}]': finished 2024-04-02T15:00:15.228 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:14 smithi169 ceph-mon[25801]: osdmap e52: 8 total, 7 up, 8 in 2024-04-02T15:00:15.228 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:14 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:15.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:14 smithi042 ceph-mon[24805]: from='client.? 172.21.15.169:0/1641578283' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "661346d4-14d3-446f-93cc-8fc8f6fd9470"}]: dispatch 2024-04-02T15:00:15.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:14 smithi042 ceph-mon[24805]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "661346d4-14d3-446f-93cc-8fc8f6fd9470"}]: dispatch 2024-04-02T15:00:15.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:14 smithi042 ceph-mon[24805]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "661346d4-14d3-446f-93cc-8fc8f6fd9470"}]': finished 2024-04-02T15:00:15.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:14 smithi042 ceph-mon[24805]: osdmap e52: 8 total, 7 up, 8 in 2024-04-02T15:00:15.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:14 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:15.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:14 smithi042 ceph-mon[29473]: from='client.? 172.21.15.169:0/1641578283' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "661346d4-14d3-446f-93cc-8fc8f6fd9470"}]: dispatch 2024-04-02T15:00:15.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:14 smithi042 ceph-mon[29473]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "661346d4-14d3-446f-93cc-8fc8f6fd9470"}]: dispatch 2024-04-02T15:00:15.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:14 smithi042 ceph-mon[29473]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "661346d4-14d3-446f-93cc-8fc8f6fd9470"}]': finished 2024-04-02T15:00:15.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:14 smithi042 ceph-mon[29473]: osdmap e52: 8 total, 7 up, 8 in 2024-04-02T15:00:15.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:14 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:16.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:16 smithi169 ceph-mon[25801]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-04-02T15:00:16.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:16 smithi169 ceph-mon[25801]: from='client.? 172.21.15.169:0/2714475921' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:00:16.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:16 smithi042 ceph-mon[29473]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-04-02T15:00:16.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:16 smithi042 ceph-mon[29473]: from='client.? 172.21.15.169:0/2714475921' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:00:16.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:16 smithi042 ceph-mon[24805]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-04-02T15:00:16.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:16 smithi042 ceph-mon[24805]: from='client.? 172.21.15.169:0/2714475921' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:00:18.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:18 smithi042 ceph-mon[24805]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T15:00:18.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:18 smithi042 ceph-mon[29473]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T15:00:18.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:18 smithi169 ceph-mon[25801]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T15:00:20.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:20 smithi169 ceph-mon[25801]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-04-02T15:00:20.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:20 smithi042 ceph-mon[24805]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-04-02T15:00:20.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:20 smithi042 ceph-mon[29473]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-04-02T15:00:21.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:21 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T15:00:21.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:21 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:21.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:21 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T15:00:21.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:21 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:21.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:21 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T15:00:21.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:21 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:22.398 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:22 smithi169 ceph-mon[25801]: Deploying daemon osd.7 on smithi169 2024-04-02T15:00:22.398 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:22 smithi169 ceph-mon[25801]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-02T15:00:22.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:22 smithi042 ceph-mon[24805]: Deploying daemon osd.7 on smithi169 2024-04-02T15:00:22.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:22 smithi042 ceph-mon[24805]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-02T15:00:22.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:22 smithi042 ceph-mon[29473]: Deploying daemon osd.7 on smithi169 2024-04-02T15:00:22.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:22 smithi042 ceph-mon[29473]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-02T15:00:24.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:24 smithi169 ceph-mon[25801]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T15:00:24.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:24 smithi042 ceph-mon[24805]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T15:00:24.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:24 smithi042 ceph-mon[29473]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T15:00:26.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:26 smithi169 ceph-mon[25801]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:26.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:26 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:26.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:26 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:26.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:26 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:26.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:26 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:26.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:26 smithi042 ceph-mon[24805]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:26.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:26 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:26.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:26 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:26.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:26 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:26.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:26 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:26.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:26 smithi042 ceph-mon[29473]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:26.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:26 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:26.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:26 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:26.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:26 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:26.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:26 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:28.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:28 smithi169 ceph-mon[25801]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:28.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:28 smithi042 ceph-mon[24805]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:28.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:28 smithi042 ceph-mon[29473]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:29.073 INFO:teuthology.orchestra.run.smithi169.stdout:Created osd(s) 7 on host 'smithi169' 2024-04-02T15:00:29.343 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:29 smithi169 ceph-mon[25801]: from='osd.7 [v2:172.21.15.169:6824/4048323867,v1:172.21.15.169:6825/4048323867]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T15:00:29.343 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:29 smithi169 ceph-mon[25801]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T15:00:29.343 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:29 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:29.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:29 smithi042 ceph-mon[24805]: from='osd.7 [v2:172.21.15.169:6824/4048323867,v1:172.21.15.169:6825/4048323867]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T15:00:29.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:29 smithi042 ceph-mon[24805]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T15:00:29.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:29 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:29.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:29 smithi042 ceph-mon[29473]: from='osd.7 [v2:172.21.15.169:6824/4048323867,v1:172.21.15.169:6825/4048323867]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T15:00:29.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:29 smithi042 ceph-mon[29473]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T15:00:29.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:29 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:29.970 DEBUG:teuthology.orchestra.run.smithi169:osd.7> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.7.service 2024-04-02T15:00:29.972 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-02T15:00:29.972 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph osd stat -f json 2024-04-02T15:00:30.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:30 smithi042 ceph-mon[24805]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:30.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:30 smithi042 ceph-mon[24805]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-02T15:00:30.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:30 smithi042 ceph-mon[24805]: osdmap e53: 8 total, 7 up, 8 in 2024-04-02T15:00:30.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:30 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:30.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:30 smithi042 ceph-mon[24805]: from='osd.7 [v2:172.21.15.169:6824/4048323867,v1:172.21.15.169:6825/4048323867]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T15:00:30.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:30 smithi042 ceph-mon[24805]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T15:00:30.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:30 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:30.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:30 smithi042 ceph-mon[29473]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:30.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:30 smithi042 ceph-mon[29473]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-02T15:00:30.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:30 smithi042 ceph-mon[29473]: osdmap e53: 8 total, 7 up, 8 in 2024-04-02T15:00:30.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:30 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:30.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:30 smithi042 ceph-mon[29473]: from='osd.7 [v2:172.21.15.169:6824/4048323867,v1:172.21.15.169:6825/4048323867]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T15:00:30.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:30 smithi042 ceph-mon[29473]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T15:00:30.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:30 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:30.506 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:00:30 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:00:30.250+0000 7fd1ff654700 -1 osd.7 0 waiting for initial osdmap 2024-04-02T15:00:30.506 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:00:30 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:00:30.258+0000 7fd1f7fe7700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T15:00:30.507 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:30 smithi169 ceph-mon[25801]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:30.507 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:30 smithi169 ceph-mon[25801]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-02T15:00:30.507 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:30 smithi169 ceph-mon[25801]: osdmap e53: 8 total, 7 up, 8 in 2024-04-02T15:00:30.507 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:30 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:30.507 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:30 smithi169 ceph-mon[25801]: from='osd.7 [v2:172.21.15.169:6824/4048323867,v1:172.21.15.169:6825/4048323867]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T15:00:30.507 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:30 smithi169 ceph-mon[25801]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2024-04-02T15:00:30.507 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:30 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:31.458 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:31 smithi042 ceph-mon[24805]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2024-04-02T15:00:31.459 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:31 smithi042 ceph-mon[24805]: osdmap e54: 8 total, 7 up, 8 in 2024-04-02T15:00:31.459 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:31.459 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:31.459 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:00:31.459 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:31 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:00:31.459 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:31 smithi042 ceph-mon[29473]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2024-04-02T15:00:31.459 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:31 smithi042 ceph-mon[29473]: osdmap e54: 8 total, 7 up, 8 in 2024-04-02T15:00:31.459 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:31 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:31.459 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:31 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:31.459 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:31 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:00:31.459 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:31 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:00:31.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:31 smithi169 ceph-mon[25801]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2024-04-02T15:00:31.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:31 smithi169 ceph-mon[25801]: osdmap e54: 8 total, 7 up, 8 in 2024-04-02T15:00:31.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:31 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:31.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:31 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:31.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:31 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:00:31.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:31 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:00:32.178 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:00:32.465 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:32 smithi042 ceph-mon[24805]: purged_snaps scrub starts 2024-04-02T15:00:32.465 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:32 smithi042 ceph-mon[24805]: purged_snaps scrub ok 2024-04-02T15:00:32.465 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:32 smithi042 ceph-mon[24805]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:32.466 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:32 smithi042 ceph-mon[24805]: osd.7 [v2:172.21.15.169:6824/4048323867,v1:172.21.15.169:6825/4048323867] boot 2024-04-02T15:00:32.466 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:32 smithi042 ceph-mon[24805]: osdmap e55: 8 total, 8 up, 8 in 2024-04-02T15:00:32.466 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:32 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:32.466 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:32 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/2002144787' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T15:00:32.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:32 smithi042 ceph-mon[29473]: purged_snaps scrub starts 2024-04-02T15:00:32.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:32 smithi042 ceph-mon[29473]: purged_snaps scrub ok 2024-04-02T15:00:32.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:32 smithi042 ceph-mon[29473]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:32.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:32 smithi042 ceph-mon[29473]: osd.7 [v2:172.21.15.169:6824/4048323867,v1:172.21.15.169:6825/4048323867] boot 2024-04-02T15:00:32.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:32 smithi042 ceph-mon[29473]: osdmap e55: 8 total, 8 up, 8 in 2024-04-02T15:00:32.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:32 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:32.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:32 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/2002144787' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T15:00:32.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:32 smithi169 ceph-mon[25801]: purged_snaps scrub starts 2024-04-02T15:00:32.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:32 smithi169 ceph-mon[25801]: purged_snaps scrub ok 2024-04-02T15:00:32.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:32 smithi169 ceph-mon[25801]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:00:32.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:32 smithi169 ceph-mon[25801]: osd.7 [v2:172.21.15.169:6824/4048323867,v1:172.21.15.169:6825/4048323867] boot 2024-04-02T15:00:32.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:32 smithi169 ceph-mon[25801]: osdmap e55: 8 total, 8 up, 8 in 2024-04-02T15:00:32.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:32 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:32.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:32 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/2002144787' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T15:00:32.814 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1712070031,"num_in_osds":8,"osd_in_since":1712070014,"num_remapped_pgs":0} 2024-04-02T15:00:32.814 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-02T15:00:33.033 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:00:33.402 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:33 smithi042 ceph-mon[24805]: osdmap e56: 8 total, 8 up, 8 in 2024-04-02T15:00:33.402 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:33 smithi042 ceph-mon[29473]: osdmap e56: 8 total, 8 up, 8 in 2024-04-02T15:00:33.612 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:33 smithi169 ceph-mon[25801]: osdmap e56: 8 total, 8 up, 8 in 2024-04-02T15:00:34.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[24805]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:34.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[24805]: osdmap e57: 8 total, 8 up, 8 in 2024-04-02T15:00:34.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[24805]: Detected new or changed devices on smithi169 2024-04-02T15:00:34.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:34.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:34.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:34.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:34.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:34.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[24805]: Adjusting osd_memory_target on smithi169 to 4272M 2024-04-02T15:00:34.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:34.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:34.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:34.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:34.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:34.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:34.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[29473]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:34.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[29473]: osdmap e57: 8 total, 8 up, 8 in 2024-04-02T15:00:34.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[29473]: Detected new or changed devices on smithi169 2024-04-02T15:00:34.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:34.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:34.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:34.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:34.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:34.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[29473]: Adjusting osd_memory_target on smithi169 to 4272M 2024-04-02T15:00:34.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:34.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:34.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:34.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:34.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:34.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:34 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:34.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:34 smithi169 ceph-mon[25801]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:34.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:34 smithi169 ceph-mon[25801]: osdmap e57: 8 total, 8 up, 8 in 2024-04-02T15:00:34.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:34 smithi169 ceph-mon[25801]: Detected new or changed devices on smithi169 2024-04-02T15:00:34.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:34 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:34.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:34 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:34.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:34 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:34.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:34 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:34.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:34 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:00:34.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:34 smithi169 ceph-mon[25801]: Adjusting osd_memory_target on smithi169 to 4272M 2024-04-02T15:00:34.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:34 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:34.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:34 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:34.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:34 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:34.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:34 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:34.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:34 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:34.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:34 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:35.040 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:00:35.041 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":58,"fsid":"fd27f2e2-f100-11ee-b647-cb9ed24678a4","created":"2024-04-02T14:55:40.720004+0000","modified":"2024-04-02T15:00:34.267661+0000","last_up_change":"2024-04-02T15:00:31.248602+0000","last_in_change":"2024-04-02T15:00:14.786180+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T14:58:39.487535+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"177da1b0-ec41-4bbe-8d6b-988afdd4cbbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6802","nonce":2441352487},{"type":"v1","addr":"172.21.15.42:6803","nonce":2441352487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6804","nonce":2441352487},{"type":"v1","addr":"172.21.15.42:6805","nonce":2441352487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6808","nonce":2441352487},{"type":"v1","addr":"172.21.15.42:6809","nonce":2441352487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6806","nonce":2441352487},{"type":"v1","addr":"172.21.15.42:6807","nonce":2441352487}]},"public_addr":"172.21.15.42:6803/2441352487","cluster_addr":"172.21.15.42:6805/2441352487","heartbeat_back_addr":"172.21.15.42:6809/2441352487","heartbeat_front_addr":"172.21.15.42:6807/2441352487","state":["exists","up"]},{"osd":1,"uuid":"aac29b7a-cf39-4dee-87e3-a905551d6230","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6810","nonce":146974053},{"type":"v1","addr":"172.21.15.42:6811","nonce":146974053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6812","nonce":146974053},{"type":"v1","addr":"172.21.15.42:6813","nonce":146974053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6816","nonce":146974053},{"type":"v1","addr":"172.21.15.42:6817","nonce":146974053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6814","nonce":146974053},{"type":"v1","addr":"172.21.15.42:6815","nonce":146974053}]},"public_addr":"172.21.15.42:6811/146974053","cluster_addr":"172.21.15.42:6813/146974053","heartbeat_back_addr":"172.21.15.42:6817/146974053","heartbeat_front_addr":"172.21.15.42:6815/146974053","state":["exists","up"]},{"osd":2,"uuid":"710ff1f2-d67f-405c-b48b-64e9fd8b6b68","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6818","nonce":744264554},{"type":"v1","addr":"172.21.15.42:6819","nonce":744264554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6820","nonce":744264554},{"type":"v1","addr":"172.21.15.42:6821","nonce":744264554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6824","nonce":744264554},{"type":"v1","addr":"172.21.15.42:6825","nonce":744264554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6822","nonce":744264554},{"type":"v1","addr":"172.21.15.42:6823","nonce":744264554}]},"public_addr":"172.21.15.42:6819/744264554","cluster_addr":"172.21.15.42:6821/744264554","heartbeat_back_addr":"172.21.15.42:6825/744264554","heartbeat_front_addr":"172.21.15.42:6823/744264554","state":["exists","up"]},{"osd":3,"uuid":"27d9b9d7-e1f8-4be8-9da8-13dd4da8dad8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6826","nonce":1633099631},{"type":"v1","addr":"172.21.15.42:6827","nonce":1633099631}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6828","nonce":1633099631},{"type":"v1","addr":"172.21.15.42:6829","nonce":1633099631}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6832","nonce":1633099631},{"type":"v1","addr":"172.21.15.42:6833","nonce":1633099631}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6830","nonce":1633099631},{"type":"v1","addr":"172.21.15.42:6831","nonce":1633099631}]},"public_addr":"172.21.15.42:6827/1633099631","cluster_addr":"172.21.15.42:6829/1633099631","heartbeat_back_addr":"172.21.15.42:6833/1633099631","heartbeat_front_addr":"172.21.15.42:6831/1633099631","state":["exists","up"]},{"osd":4,"uuid":"32958af8-ae8a-4abc-95d9-7c08ac3f758e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":2353291645},{"type":"v1","addr":"172.21.15.169:6801","nonce":2353291645}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":2353291645},{"type":"v1","addr":"172.21.15.169:6803","nonce":2353291645}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":2353291645},{"type":"v1","addr":"172.21.15.169:6807","nonce":2353291645}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":2353291645},{"type":"v1","addr":"172.21.15.169:6805","nonce":2353291645}]},"public_addr":"172.21.15.169:6801/2353291645","cluster_addr":"172.21.15.169:6803/2353291645","heartbeat_back_addr":"172.21.15.169:6807/2353291645","heartbeat_front_addr":"172.21.15.169:6805/2353291645","state":["exists","up"]},{"osd":5,"uuid":"46d22890-4110-43a7-9c0d-7406495a9924","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":1262879371},{"type":"v1","addr":"172.21.15.169:6809","nonce":1262879371}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":1262879371},{"type":"v1","addr":"172.21.15.169:6811","nonce":1262879371}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":1262879371},{"type":"v1","addr":"172.21.15.169:6815","nonce":1262879371}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":1262879371},{"type":"v1","addr":"172.21.15.169:6813","nonce":1262879371}]},"public_addr":"172.21.15.169:6809/1262879371","cluster_addr":"172.21.15.169:6811/1262879371","heartbeat_back_addr":"172.21.15.169:6815/1262879371","heartbeat_front_addr":"172.21.15.169:6813/1262879371","state":["exists","up"]},{"osd":6,"uuid":"aca7c09b-7ae9-45b2-91d2-48413a58886e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":3799859416},{"type":"v1","addr":"172.21.15.169:6817","nonce":3799859416}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":3799859416},{"type":"v1","addr":"172.21.15.169:6819","nonce":3799859416}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6822","nonce":3799859416},{"type":"v1","addr":"172.21.15.169:6823","nonce":3799859416}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":3799859416},{"type":"v1","addr":"172.21.15.169:6821","nonce":3799859416}]},"public_addr":"172.21.15.169:6817/3799859416","cluster_addr":"172.21.15.169:6819/3799859416","heartbeat_back_addr":"172.21.15.169:6823/3799859416","heartbeat_front_addr":"172.21.15.169:6821/3799859416","state":["exists","up"]},{"osd":7,"uuid":"661346d4-14d3-446f-93cc-8fc8f6fd9470","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6824","nonce":4048323867},{"type":"v1","addr":"172.21.15.169:6825","nonce":4048323867}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6826","nonce":4048323867},{"type":"v1","addr":"172.21.15.169:6827","nonce":4048323867}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6830","nonce":4048323867},{"type":"v1","addr":"172.21.15.169:6831","nonce":4048323867}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6828","nonce":4048323867},{"type":"v1","addr":"172.21.15.169:6829","nonce":4048323867}]},"public_addr":"172.21.15.169:6825/4048323867","cluster_addr":"172.21.15.169:6827/4048323867","heartbeat_back_addr":"172.21.15.169:6831/4048323867","heartbeat_front_addr":"172.21.15.169:6829/4048323867","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:57:48.227447+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:58:12.105585+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:58:35.642777+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:58:59.867595+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:59:20.837579+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:59:43.638559+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:00:06.825493+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:00:30.029025+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.42:0/3168280765":"2024-04-03T14:56:30.584985+0000","172.21.15.42:0/1443281804":"2024-04-03T14:56:30.584985+0000","172.21.15.42:6801/1586110338":"2024-04-03T14:56:30.584985+0000","172.21.15.42:6800/1586110338":"2024-04-03T14:56:30.584985+0000","172.21.15.42:0/3226246413":"2024-04-03T14:56:30.584985+0000","172.21.15.42:0/1870022407":"2024-04-03T14:56:08.269630+0000","172.21.15.42:0/3766045960":"2024-04-03T14:56:08.269630+0000","172.21.15.42:0/1412464921":"2024-04-03T14:56:08.269630+0000","172.21.15.42:6800/3054125886":"2024-04-03T14:56:08.269630+0000","172.21.15.42:6801/3054125886":"2024-04-03T14:56:08.269630+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T15:00:35.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:35 smithi042 ceph-mon[24805]: osdmap e58: 8 total, 8 up, 8 in 2024-04-02T15:00:35.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:35 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3308913705' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:00:35.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:35 smithi042 ceph-mon[29473]: osdmap e58: 8 total, 8 up, 8 in 2024-04-02T15:00:35.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:35 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3308913705' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:00:35.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:35 smithi169 ceph-mon[25801]: osdmap e58: 8 total, 8 up, 8 in 2024-04-02T15:00:35.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:35 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3308913705' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:00:35.817 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-02T14:58:39.487535+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-02T15:00:35.817 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph osd pool get .mgr pg_num 2024-04-02T15:00:36.041 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:00:36.425 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:36 smithi042 ceph-mon[24805]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:36.425 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:36 smithi042 ceph-mon[24805]: osdmap e59: 8 total, 8 up, 8 in 2024-04-02T15:00:36.426 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:36 smithi042 ceph-mon[29473]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:36.426 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:36 smithi042 ceph-mon[29473]: osdmap e59: 8 total, 8 up, 8 in 2024-04-02T15:00:36.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:36 smithi169 ceph-mon[25801]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:36.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:36 smithi169 ceph-mon[25801]: osdmap e59: 8 total, 8 up, 8 in 2024-04-02T15:00:37.780 INFO:teuthology.orchestra.run.smithi042.stdout:pg_num: 1 2024-04-02T15:00:38.480 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:38 smithi042 ceph-mon[24805]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:38.481 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:38 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/4199801998' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-02T15:00:38.481 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:38 smithi042 ceph-mon[29473]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:38.481 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:38 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/4199801998' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-02T15:00:38.507 INFO:tasks.cephadm:Adding prometheus.a on smithi169 2024-04-02T15:00:38.507 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch apply prometheus '1;smithi169=a' 2024-04-02T15:00:38.538 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:38 smithi169 ceph-mon[25801]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:38.538 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:38 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/4199801998' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-02T15:00:40.421 INFO:teuthology.orchestra.run.smithi169.stdout:Scheduled prometheus update... 2024-04-02T15:00:40.422 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:40 smithi169 ceph-mon[25801]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T15:00:40.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:40 smithi042 ceph-mon[24805]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T15:00:40.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:40 smithi042 ceph-mon[29473]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T15:00:40.982 DEBUG:teuthology.orchestra.run.smithi169:prometheus.a> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@prometheus.a.service 2024-04-02T15:00:41.024 INFO:tasks.cephadm:Adding node-exporter.a on smithi042 2024-04-02T15:00:41.025 INFO:tasks.cephadm:Adding node-exporter.b on smithi169 2024-04-02T15:00:41.025 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch apply node-exporter '2;smithi042=a;smithi169=b' 2024-04-02T15:00:41.570 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:41 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ignoring --setuser ceph since I am not root 2024-04-02T15:00:41.570 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:41 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ignoring --setgroup ceph since I am not root 2024-04-02T15:00:41.570 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:41 smithi169 ceph-mgr[26985]: -- 172.21.15.169:0/455295970 <== mon.2 v2:172.21.15.169:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b28faf2340 con 0x55b28facb000 2024-04-02T15:00:41.571 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:41 smithi169 ceph-mon[25801]: from='client.24329 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi169=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:00:41.571 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:41 smithi169 ceph-mon[25801]: Saving service prometheus spec with placement smithi169=a;count:1 2024-04-02T15:00:41.571 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:41 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:41.571 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:41 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:41.571 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:41 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:41.571 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:41 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:41.571 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:41 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:41.571 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:41 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-02T15:00:41.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[24805]: from='client.24329 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi169=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:00:41.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[24805]: Saving service prometheus spec with placement smithi169=a;count:1 2024-04-02T15:00:41.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:41.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:41.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:41.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:41.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:41.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-02T15:00:41.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[29473]: from='client.24329 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi169=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:00:41.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[29473]: Saving service prometheus spec with placement smithi169=a;count:1 2024-04-02T15:00:41.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:41.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:41.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:41.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:41.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' 2024-04-02T15:00:41.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-02T15:00:41.795 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ignoring --setuser ceph since I am not root 2024-04-02T15:00:41.795 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ignoring --setgroup ceph since I am not root 2024-04-02T15:00:41.795 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-mgr[25052]: -- 172.21.15.42:0/1669355182 <== mon.0 v2:172.21.15.42:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55be51248340 con 0x55be51220c00 2024-04-02T15:00:42.227 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:41 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:41.977+0000 7f23476f0000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T15:00:42.228 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:42 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:42.070+0000 7f23476f0000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T15:00:42.293 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:41.974+0000 7f0e2f6ce000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T15:00:42.293 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:42 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:42.065+0000 7f0e2f6ce000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T15:00:42.603 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:42 smithi169 ceph-mon[25801]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-02T15:00:42.603 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:42 smithi169 ceph-mon[25801]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-02T15:00:42.713 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:42 smithi042 ceph-mon[24805]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-02T15:00:42.714 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:42 smithi042 ceph-mon[24805]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-02T15:00:42.714 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:42 smithi042 ceph-mon[29473]: from='mgr.14148 172.21.15.42:0/1255655491' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-02T15:00:42.714 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:42 smithi042 ceph-mon[29473]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-02T15:00:42.714 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:42 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:42.668+0000 7f0e2f6ce000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T15:00:42.871 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:42 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:42.686+0000 7f23476f0000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T15:00:42.871 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:42 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:42.777+0000 7f23476f0000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T15:00:42.871 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:42 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:42.881+0000 7f23476f0000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T15:00:43.002 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:42 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:42.775+0000 7f0e2f6ce000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T15:00:43.002 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:42 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:42.860+0000 7f0e2f6ce000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T15:00:43.002 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:43.010+0000 7f0e2f6ce000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T15:00:43.210 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:43 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:43.034+0000 7f23476f0000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T15:00:43.210 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:43 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:43.114+0000 7f23476f0000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T15:00:43.210 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:43 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:43.220+0000 7f23476f0000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T15:00:43.280 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:43.092+0000 7f0e2f6ce000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T15:00:43.280 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:43.203+0000 7f0e2f6ce000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T15:00:43.280 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:43.291+0000 7f0e2f6ce000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T15:00:43.477 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:43 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:43.305+0000 7f23476f0000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T15:00:43.477 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:43 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:43.468+0000 7f23476f0000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T15:00:43.532 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:43.456+0000 7f0e2f6ce000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T15:00:43.793 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:43.543+0000 7f0e2f6ce000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T15:00:43.977 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:43 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:43.555+0000 7f23476f0000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T15:00:44.885 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:44 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:44.577+0000 7f23476f0000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T15:00:44.885 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:44 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:44.706+0000 7f23476f0000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T15:00:44.885 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:44 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:44.895+0000 7f23476f0000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T15:00:44.902 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:44 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:44.586+0000 7f0e2f6ce000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T15:00:44.903 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:44 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:44.718+0000 7f0e2f6ce000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T15:00:44.903 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:44 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:44.913+0000 7f0e2f6ce000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T15:00:45.227 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:44 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:44.982+0000 7f23476f0000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T15:00:45.227 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:45 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:45.117+0000 7f23476f0000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:00:45.293 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:45 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:45.003+0000 7f0e2f6ce000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T15:00:45.293 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:45 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:45.142+0000 7f0e2f6ce000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:00:45.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:45 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:45.435+0000 7f23476f0000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T15:00:45.728 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:45 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:45.655+0000 7f23476f0000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:00:45.770 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:45 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:45.466+0000 7f0e2f6ce000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T15:00:45.770 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:45 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:45.694+0000 7f0e2f6ce000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:00:45.770 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:45 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:45.781+0000 7f0e2f6ce000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T15:00:46.043 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:45 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:45.955+0000 7f0e2f6ce000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T15:00:46.227 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:45 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:45.739+0000 7f23476f0000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T15:00:46.227 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:45 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:45.908+0000 7f23476f0000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T15:00:47.102 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:46 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:46.820+0000 7f23476f0000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T15:00:47.214 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:46 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:46.917+0000 7f0e2f6ce000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T15:00:47.402 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:47 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:47.112+0000 7f23476f0000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T15:00:47.402 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:47 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:47.206+0000 7f23476f0000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T15:00:47.402 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:47 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:47.353+0000 7f23476f0000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T15:00:47.403 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:47 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: [02/Apr/2024:15:00:47] ENGINE Bus STARTING 2024-04-02T15:00:47.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:47 smithi042 ceph-mon[29473]: Standby manager daemon x restarted 2024-04-02T15:00:47.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:47 smithi042 ceph-mon[29473]: Standby manager daemon x started 2024-04-02T15:00:47.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:47 smithi042 ceph-mon[29473]: from='mgr.? 172.21.15.169:0/2851694164' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T15:00:47.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:47 smithi042 ceph-mon[29473]: from='mgr.? 172.21.15.169:0/2851694164' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T15:00:47.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:47 smithi042 ceph-mon[29473]: from='mgr.? 172.21.15.169:0/2851694164' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T15:00:47.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:47 smithi042 ceph-mon[29473]: from='mgr.? 172.21.15.169:0/2851694164' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T15:00:47.479 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:47 smithi042 ceph-mon[24805]: Standby manager daemon x restarted 2024-04-02T15:00:47.479 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:47 smithi042 ceph-mon[24805]: Standby manager daemon x started 2024-04-02T15:00:47.479 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:47 smithi042 ceph-mon[24805]: from='mgr.? 172.21.15.169:0/2851694164' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T15:00:47.479 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:47 smithi042 ceph-mon[24805]: from='mgr.? 172.21.15.169:0/2851694164' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T15:00:47.479 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:47 smithi042 ceph-mon[24805]: from='mgr.? 172.21.15.169:0/2851694164' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T15:00:47.479 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:47 smithi042 ceph-mon[24805]: from='mgr.? 172.21.15.169:0/2851694164' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T15:00:47.479 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:47 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:47.225+0000 7f0e2f6ce000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T15:00:47.479 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:47 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:47.325+0000 7f0e2f6ce000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T15:00:47.479 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:47 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:47.488+0000 7f0e2f6ce000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T15:00:47.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:47 smithi169 ceph-mon[25801]: Standby manager daemon x restarted 2024-04-02T15:00:47.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:47 smithi169 ceph-mon[25801]: Standby manager daemon x started 2024-04-02T15:00:47.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:47 smithi169 ceph-mon[25801]: from='mgr.? 172.21.15.169:0/2851694164' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T15:00:47.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:47 smithi169 ceph-mon[25801]: from='mgr.? 172.21.15.169:0/2851694164' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T15:00:47.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:47 smithi169 ceph-mon[25801]: from='mgr.? 172.21.15.169:0/2851694164' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T15:00:47.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:47 smithi169 ceph-mon[25801]: from='mgr.? 172.21.15.169:0/2851694164' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T15:00:47.729 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:47 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: CherryPy Checker: 2024-04-02T15:00:47.729 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:47 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: The Application mounted at '' has an empty config. 2024-04-02T15:00:47.729 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:47 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: 2024-04-02T15:00:47.729 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:47 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: [02/Apr/2024:15:00:47] ENGINE Serving on http://:::9283 2024-04-02T15:00:47.729 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:00:47 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: [02/Apr/2024:15:00:47] ENGINE Bus STARTED 2024-04-02T15:00:48.466 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:48 smithi042 ceph-mon[24805]: mgrmap e17: y(active, since 4m), standbys: x 2024-04-02T15:00:48.466 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:48 smithi042 ceph-mon[24805]: Active manager daemon y restarted 2024-04-02T15:00:48.466 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:48 smithi042 ceph-mon[24805]: Activating manager daemon y 2024-04-02T15:00:48.466 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:48 smithi042 ceph-mon[24805]: osdmap e60: 8 total, 8 up, 8 in 2024-04-02T15:00:48.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:48 smithi042 ceph-mon[29473]: mgrmap e17: y(active, since 4m), standbys: x 2024-04-02T15:00:48.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:48 smithi042 ceph-mon[29473]: Active manager daemon y restarted 2024-04-02T15:00:48.467 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:48 smithi042 ceph-mon[29473]: Activating manager daemon y 2024-04-02T15:00:48.467 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:48 smithi042 ceph-mon[29473]: osdmap e60: 8 total, 8 up, 8 in 2024-04-02T15:00:48.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:48 smithi169 ceph-mon[25801]: mgrmap e17: y(active, since 4m), standbys: x 2024-04-02T15:00:48.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:48 smithi169 ceph-mon[25801]: Active manager daemon y restarted 2024-04-02T15:00:48.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:48 smithi169 ceph-mon[25801]: Activating manager daemon y 2024-04-02T15:00:48.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:48 smithi169 ceph-mon[25801]: osdmap e60: 8 total, 8 up, 8 in 2024-04-02T15:00:48.729 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:48 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: [02/Apr/2024:15:00:48] ENGINE Bus STARTING 2024-04-02T15:00:48.729 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:48 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: CherryPy Checker: 2024-04-02T15:00:48.991 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:48 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: The Application mounted at '' has an empty config. 2024-04-02T15:00:48.991 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:48 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:00:48.992 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:48 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: [02/Apr/2024:15:00:48] ENGINE Serving on http://:::9283 2024-04-02T15:00:48.992 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:48 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: [02/Apr/2024:15:00:48] ENGINE Bus STARTED 2024-04-02T15:00:49.435 INFO:teuthology.orchestra.run.smithi169.stdout:Scheduled node-exporter update... 2024-04-02T15:00:49.709 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: mgrmap e18: y(active, starting, since 0.928845s), standbys: x 2024-04-02T15:00:49.709 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:00:49.709 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:00:49.709 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:00:49.709 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T15:00:49.709 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T15:00:49.709 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:00:49.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:00:49.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:00:49.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:00:49.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:00:49.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:00:49.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:00:49.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:49.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T15:00:49.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T15:00:49.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T15:00:49.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: Manager daemon y is now available 2024-04-02T15:00:49.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:49.710 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:49.711 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:49.711 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:49.711 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:49.711 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:00:49.711 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:00:49.711 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:00:49.711 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:00:49.712 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: mgrmap e18: y(active, starting, since 0.928845s), standbys: x 2024-04-02T15:00:49.712 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:00:49.712 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:00:49.712 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:00:49.712 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T15:00:49.712 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T15:00:49.712 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:00:49.712 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:00:49.712 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:00:49.712 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:00:49.712 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:00:49.713 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:00:49.713 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:00:49.713 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:49.713 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T15:00:49.713 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T15:00:49.713 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T15:00:49.713 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: Manager daemon y is now available 2024-04-02T15:00:49.713 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:49.713 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:49.713 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:49.713 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:49.713 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:49.714 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:00:49.714 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:00:49.714 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:00:49.714 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:00:49.714 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: [02/Apr/2024:15:00:49] ENGINE Bus STARTING 2024-04-02T15:00:49.714 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: [02/Apr/2024:15:00:49] ENGINE Serving on https://172.21.15.42:7150 2024-04-02T15:00:49.714 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:00:49 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: [02/Apr/2024:15:00:49] ENGINE Bus STARTED 2024-04-02T15:00:49.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: mgrmap e18: y(active, starting, since 0.928845s), standbys: x 2024-04-02T15:00:49.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:00:49.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:00:49.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:00:49.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T15:00:49.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T15:00:49.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:00:49.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:00:49.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:00:49.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:00:49.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:00:49.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:00:49.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:00:49.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:00:49.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T15:00:49.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T15:00:49.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T15:00:49.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: Manager daemon y is now available 2024-04-02T15:00:49.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:49.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:00:49.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:00:49.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:00:49.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:49.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:00:49.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:00:49.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:00:49.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:49 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:00:50.427 DEBUG:teuthology.orchestra.run.smithi042:node-exporter.a> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@node-exporter.a.service 2024-04-02T15:00:50.430 DEBUG:teuthology.orchestra.run.smithi169:node-exporter.b> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@node-exporter.b.service 2024-04-02T15:00:50.432 INFO:tasks.cephadm:Adding alertmanager.a on smithi042 2024-04-02T15:00:50.433 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch apply alertmanager '1;smithi042=a' 2024-04-02T15:00:50.669 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:50 smithi169 ceph-mon[25801]: mgrmap e19: y(active, since 1.94274s), standbys: x 2024-04-02T15:00:50.669 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:50 smithi169 ceph-mon[25801]: Saving service node-exporter spec with placement smithi042=a;smithi169=b;count:2 2024-04-02T15:00:50.669 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:50 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:50.669 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:50 smithi169 ceph-mon[25801]: [02/Apr/2024:15:00:49] ENGINE Bus STARTING 2024-04-02T15:00:50.669 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:50 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:50.669 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:50 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:50.751 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:50 smithi042 ceph-mon[29473]: mgrmap e19: y(active, since 1.94274s), standbys: x 2024-04-02T15:00:50.751 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:50 smithi042 ceph-mon[29473]: Saving service node-exporter spec with placement smithi042=a;smithi169=b;count:2 2024-04-02T15:00:50.752 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:50 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:50.752 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:50 smithi042 ceph-mon[29473]: [02/Apr/2024:15:00:49] ENGINE Bus STARTING 2024-04-02T15:00:50.752 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:50 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:50.752 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:50 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:50.752 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:50 smithi042 ceph-mon[24805]: mgrmap e19: y(active, since 1.94274s), standbys: x 2024-04-02T15:00:50.752 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:50 smithi042 ceph-mon[24805]: Saving service node-exporter spec with placement smithi042=a;smithi169=b;count:2 2024-04-02T15:00:50.752 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:50 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:50.752 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:50 smithi042 ceph-mon[24805]: [02/Apr/2024:15:00:49] ENGINE Bus STARTING 2024-04-02T15:00:50.753 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:50 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:50.753 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:50 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:51.641 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:51 smithi169 ceph-mon[25801]: [02/Apr/2024:15:00:49] ENGINE Serving on https://172.21.15.42:7150 2024-04-02T15:00:51.641 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:51 smithi169 ceph-mon[25801]: [02/Apr/2024:15:00:49] ENGINE Bus STARTED 2024-04-02T15:00:51.641 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:51 smithi169 ceph-mon[25801]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:51.641 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:51 smithi169 ceph-mon[25801]: mgrmap e20: y(active, since 2s), standbys: x 2024-04-02T15:00:51.642 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:51 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:51.642 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:51 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:51.642 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:51 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:51.642 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:51 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:51.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[24805]: [02/Apr/2024:15:00:49] ENGINE Serving on https://172.21.15.42:7150 2024-04-02T15:00:51.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[24805]: [02/Apr/2024:15:00:49] ENGINE Bus STARTED 2024-04-02T15:00:51.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[24805]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:51.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[24805]: mgrmap e20: y(active, since 2s), standbys: x 2024-04-02T15:00:51.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:51.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:51.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:51.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:51.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[29473]: [02/Apr/2024:15:00:49] ENGINE Serving on https://172.21.15.42:7150 2024-04-02T15:00:51.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[29473]: [02/Apr/2024:15:00:49] ENGINE Bus STARTED 2024-04-02T15:00:51.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[29473]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:51.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[29473]: mgrmap e20: y(active, since 2s), standbys: x 2024-04-02T15:00:51.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:51.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:51.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:51.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:51 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:52.590 INFO:teuthology.orchestra.run.smithi169.stdout:Scheduled alertmanager update... 2024-04-02T15:00:52.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:52 smithi169 ceph-mon[25801]: Updating smithi042:/etc/ceph/ceph.conf 2024-04-02T15:00:52.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:52 smithi169 ceph-mon[25801]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:00:52.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:52 smithi169 ceph-mon[25801]: Updating smithi169:/etc/ceph/ceph.conf 2024-04-02T15:00:52.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:52 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:52.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:52 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:52.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:52 smithi042 ceph-mon[24805]: Updating smithi042:/etc/ceph/ceph.conf 2024-04-02T15:00:52.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:52 smithi042 ceph-mon[24805]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:00:52.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:52 smithi042 ceph-mon[24805]: Updating smithi169:/etc/ceph/ceph.conf 2024-04-02T15:00:52.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:52 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:52.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:52 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:52.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:52 smithi042 ceph-mon[29473]: Updating smithi042:/etc/ceph/ceph.conf 2024-04-02T15:00:52.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:52 smithi042 ceph-mon[29473]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:00:52.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:52 smithi042 ceph-mon[29473]: Updating smithi169:/etc/ceph/ceph.conf 2024-04-02T15:00:52.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:52 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:52.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:52 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:53.539 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:53 smithi169 ceph-mon[25801]: Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:00:53.540 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:53 smithi169 ceph-mon[25801]: Deploying daemon prometheus.a on smithi169 2024-04-02T15:00:53.540 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:53 smithi169 ceph-mon[25801]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:53.540 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:53 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:53.563 DEBUG:teuthology.orchestra.run.smithi042:alertmanager.a> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@alertmanager.a.service 2024-04-02T15:00:53.565 INFO:tasks.cephadm:Adding grafana.a on smithi169 2024-04-02T15:00:53.566 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph orch apply grafana '1;smithi169=a' 2024-04-02T15:00:53.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:53 smithi042 ceph-mon[29473]: Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:00:53.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:53 smithi042 ceph-mon[29473]: Deploying daemon prometheus.a on smithi169 2024-04-02T15:00:53.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:53 smithi042 ceph-mon[29473]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:53.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:53 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:53.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:53 smithi042 ceph-mon[24805]: Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:00:53.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:53 smithi042 ceph-mon[24805]: Deploying daemon prometheus.a on smithi169 2024-04-02T15:00:53.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:53 smithi042 ceph-mon[24805]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:53.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:53 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:54.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:54 smithi169 ceph-mon[25801]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi042=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:00:54.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:54 smithi169 ceph-mon[25801]: Saving service alertmanager spec with placement smithi042=a;count:1 2024-04-02T15:00:54.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:54 smithi042 ceph-mon[24805]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi042=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:00:54.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:54 smithi042 ceph-mon[24805]: Saving service alertmanager spec with placement smithi042=a;count:1 2024-04-02T15:00:54.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:54 smithi042 ceph-mon[29473]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi042=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:00:54.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:54 smithi042 ceph-mon[29473]: Saving service alertmanager spec with placement smithi042=a;count:1 2024-04-02T15:00:55.696 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:55 smithi169 ceph-mon[25801]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:55.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:55 smithi042 ceph-mon[24805]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:55.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:55 smithi042 ceph-mon[29473]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:56.393 INFO:teuthology.orchestra.run.smithi169.stdout:Scheduled grafana update... 2024-04-02T15:00:57.367 DEBUG:teuthology.orchestra.run.smithi169:grafana.a> sudo journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@grafana.a.service 2024-04-02T15:00:57.369 INFO:tasks.cephadm:Setting up client nodes... 2024-04-02T15:00:57.372 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-02T15:00:57.689 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:57 smithi042 ceph-mon[24805]: from='client.24380 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi169=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:00:57.689 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:57 smithi042 ceph-mon[24805]: Saving service grafana spec with placement smithi169=a;count:1 2024-04-02T15:00:57.689 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:57 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:57.690 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:57 smithi042 ceph-mon[24805]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:57.690 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:57 smithi042 ceph-mon[29473]: from='client.24380 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi169=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:00:57.690 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:57 smithi042 ceph-mon[29473]: Saving service grafana spec with placement smithi169=a;count:1 2024-04-02T15:00:57.690 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:57 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:57.690 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:57 smithi042 ceph-mon[29473]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:57.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:57 smithi169 ceph-mon[25801]: from='client.24380 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi169=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:00:57.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:57 smithi169 ceph-mon[25801]: Saving service grafana spec with placement smithi169=a;count:1 2024-04-02T15:00:57.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:57 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:00:57.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:57 smithi169 ceph-mon[25801]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:59.491 INFO:teuthology.orchestra.run.smithi042.stdout:[client.0] 2024-04-02T15:00:59.491 INFO:teuthology.orchestra.run.smithi042.stdout: key = AQCrHQxmHUjDHRAAnCAOTnDg59tPJdctKFMMBg== 2024-04-02T15:00:59.724 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:00:59 smithi042 ceph-mon[24805]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:59.724 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:00:59 smithi042 ceph-mon[29473]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:00:59.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:00:59 smithi169 ceph-mon[25801]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:00.064 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-04-02T15:01:00.065 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-02T15:01:00.065 DEBUG:teuthology.orchestra.run.smithi042:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-02T15:01:00.109 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-02T15:01:00.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:00 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1298011383' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:01:00.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:00 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:01:00.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:00 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T15:01:00.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:00 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1298011383' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:01:00.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:00 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:01:00.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:00 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T15:01:00.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:00 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1298011383' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:01:00.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:00 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:01:00.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:00 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T15:01:01.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:01 smithi042 ceph-mon[24805]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:01.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:01 smithi042 ceph-mon[29473]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:01.608 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:01 smithi169 ceph-mon[25801]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:02.908 INFO:teuthology.orchestra.run.smithi169.stdout:[client.1] 2024-04-02T15:01:02.909 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQCuHQxmuEGXNhAAdkJvJK/AKq/qNdnBppznMQ== 2024-04-02T15:01:03.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:03 smithi169 ceph-mon[25801]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:03.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:03 smithi169 ceph-mon[25801]: from='client.? 172.21.15.169:0/84627970' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:01:03.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:03 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:01:03.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:03 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T15:01:03.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:03 smithi042 ceph-mon[24805]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:03.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:03 smithi042 ceph-mon[24805]: from='client.? 172.21.15.169:0/84627970' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:01:03.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:03 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:01:03.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:03 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T15:01:03.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:03 smithi042 ceph-mon[29473]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:03.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:03 smithi042 ceph-mon[29473]: from='client.? 172.21.15.169:0/84627970' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:01:03.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:03 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:01:03.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:03 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T15:01:04.629 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-04-02T15:01:04.629 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-02T15:01:04.629 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-02T15:01:04.677 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-02T15:01:04.677 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-02T15:01:04.677 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-02T15:01:04.906 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:05.474 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 systemd[1]: Starting Ceph prometheus.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:01:05.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-mon[25801]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:05.799 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 podman[40931]: 2024-04-02 15:01:05.489077292 +0000 UTC m=+0.018964756 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-02T15:01:05.799 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 podman[40931]: 2024-04-02 15:01:05.592798075 +0000 UTC m=+0.122685557 container create 44d2bff3219178a41210ffaa2cc68b596cf1e8aac20725b55a8668c9bf0edbc0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:01:05.800 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:05 smithi042 ceph-mon[24805]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:05.800 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:05 smithi042 ceph-mon[29473]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:06.228 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 podman[40931]: 2024-04-02 15:01:05.855385847 +0000 UTC m=+0.385273296 container init 44d2bff3219178a41210ffaa2cc68b596cf1e8aac20725b55a8668c9bf0edbc0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:01:06.228 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 podman[40931]: 2024-04-02 15:01:05.857836122 +0000 UTC m=+0.387723569 container start 44d2bff3219178a41210ffaa2cc68b596cf1e8aac20725b55a8668c9bf0edbc0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:01:06.228 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.882Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-02T15:01:06.228 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.882Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-02T15:01:06.228 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.882Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-02T15:01:06.228 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.882Z caller=main.go:518 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi169 (none))" 2024-04-02T15:01:06.228 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.882Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-02T15:01:06.229 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.882Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-02T15:01:06.229 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.883Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-02T15:01:06.229 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.883Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-02T15:01:06.229 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.885Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-02T15:01:06.229 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.886Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-02T15:01:06.229 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.886Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.667µs 2024-04-02T15:01:06.229 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.886Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-02T15:01:06.229 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.886Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-02T15:01:06.229 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.887Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=23.022µs wal_replay_duration=144.163µs total_replay_duration=181.737µs 2024-04-02T15:01:06.229 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.887Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-02T15:01:06.229 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.887Z caller=main.go:947 level=info msg="TSDB started" 2024-04-02T15:01:06.230 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.887Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-02T15:01:06.230 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.903Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.892351ms db_storage=612ns remote_storage=1.455µs web_handler=282ns query_engine=537ns scrape=579.029µs scrape_sd=21.878µs notify=876ns notify_sd=1.961µs rules=15.131636ms 2024-04-02T15:01:06.230 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:01:05.903Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-02T15:01:06.230 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 bash[40931]: 44d2bff3219178a41210ffaa2cc68b596cf1e8aac20725b55a8668c9bf0edbc0 2024-04-02T15:01:06.230 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:01:05 smithi169 systemd[1]: Started Ceph prometheus.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T15:01:06.897 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:01:07.248 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:07 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:07.248 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:07 smithi042 ceph-mon[24805]: Deploying daemon node-exporter.a on smithi042 2024-04-02T15:01:07.248 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:07 smithi042 ceph-mon[24805]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:07.248 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:07 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3590565679' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:01:07.249 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:07 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:07.249 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:07 smithi042 ceph-mon[29473]: Deploying daemon node-exporter.a on smithi042 2024-04-02T15:01:07.249 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:07 smithi042 ceph-mon[29473]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:07.249 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:07 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3590565679' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:01:07.249 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:07 smithi042 systemd[1]: Starting Ceph node-exporter.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:01:07.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:07 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:07.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:07 smithi169 ceph-mon[25801]: Deploying daemon node-exporter.a on smithi042 2024-04-02T15:01:07.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:07 smithi169 ceph-mon[25801]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:07.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:07 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3590565679' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:01:07.562 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":20,"active_gid":14442,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6800","nonce":2756577671},{"type":"v1","addr":"172.21.15.42:6801","nonce":2756577671}]},"active_addr":"172.21.15.42:6801/2756577671","active_change":"2024-04-02T15:00:47.490942+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":24335,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.42:8443/","prometheus":"http://172.21.15.42:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":60,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":3176984497}]},{"addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":975868041}]},{"addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":333974306}]},{"addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":2580100108}]}]}} 2024-04-02T15:01:07.566 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-02T15:01:07.566 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-02T15:01:07.566 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-02T15:01:07.793 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:07 smithi042 bash[44285]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-04-02T15:01:07.808 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:08.543 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:08 smithi042 bash[44285]: Getting image source signatures 2024-04-02T15:01:08.543 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:08 smithi042 bash[44285]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-04-02T15:01:08.543 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:08 smithi042 bash[44285]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-02T15:01:08.543 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:08 smithi042 bash[44285]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-04-02T15:01:09.690 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:09 smithi042 ceph-mon[24805]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:09.690 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:09 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:09.690 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:09 smithi042 ceph-mon[29473]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:09.691 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:09 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:09.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:09 smithi169 ceph-mon[25801]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:09.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:09 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:09.861 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:01:09.861 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":60,"fsid":"fd27f2e2-f100-11ee-b647-cb9ed24678a4","created":"2024-04-02T14:55:40.720004+0000","modified":"2024-04-02T15:00:47.490511+0000","last_up_change":"2024-04-02T15:00:31.248602+0000","last_in_change":"2024-04-02T15:00:14.786180+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T14:58:39.487535+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"177da1b0-ec41-4bbe-8d6b-988afdd4cbbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6802","nonce":2441352487},{"type":"v1","addr":"172.21.15.42:6803","nonce":2441352487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6804","nonce":2441352487},{"type":"v1","addr":"172.21.15.42:6805","nonce":2441352487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6808","nonce":2441352487},{"type":"v1","addr":"172.21.15.42:6809","nonce":2441352487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6806","nonce":2441352487},{"type":"v1","addr":"172.21.15.42:6807","nonce":2441352487}]},"public_addr":"172.21.15.42:6803/2441352487","cluster_addr":"172.21.15.42:6805/2441352487","heartbeat_back_addr":"172.21.15.42:6809/2441352487","heartbeat_front_addr":"172.21.15.42:6807/2441352487","state":["exists","up"]},{"osd":1,"uuid":"aac29b7a-cf39-4dee-87e3-a905551d6230","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6810","nonce":146974053},{"type":"v1","addr":"172.21.15.42:6811","nonce":146974053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6812","nonce":146974053},{"type":"v1","addr":"172.21.15.42:6813","nonce":146974053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6816","nonce":146974053},{"type":"v1","addr":"172.21.15.42:6817","nonce":146974053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6814","nonce":146974053},{"type":"v1","addr":"172.21.15.42:6815","nonce":146974053}]},"public_addr":"172.21.15.42:6811/146974053","cluster_addr":"172.21.15.42:6813/146974053","heartbeat_back_addr":"172.21.15.42:6817/146974053","heartbeat_front_addr":"172.21.15.42:6815/146974053","state":["exists","up"]},{"osd":2,"uuid":"710ff1f2-d67f-405c-b48b-64e9fd8b6b68","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6818","nonce":744264554},{"type":"v1","addr":"172.21.15.42:6819","nonce":744264554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6820","nonce":744264554},{"type":"v1","addr":"172.21.15.42:6821","nonce":744264554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6824","nonce":744264554},{"type":"v1","addr":"172.21.15.42:6825","nonce":744264554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6822","nonce":744264554},{"type":"v1","addr":"172.21.15.42:6823","nonce":744264554}]},"public_addr":"172.21.15.42:6819/744264554","cluster_addr":"172.21.15.42:6821/744264554","heartbeat_back_addr":"172.21.15.42:6825/744264554","heartbeat_front_addr":"172.21.15.42:6823/744264554","state":["exists","up"]},{"osd":3,"uuid":"27d9b9d7-e1f8-4be8-9da8-13dd4da8dad8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6826","nonce":1633099631},{"type":"v1","addr":"172.21.15.42:6827","nonce":1633099631}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6828","nonce":1633099631},{"type":"v1","addr":"172.21.15.42:6829","nonce":1633099631}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6832","nonce":1633099631},{"type":"v1","addr":"172.21.15.42:6833","nonce":1633099631}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6830","nonce":1633099631},{"type":"v1","addr":"172.21.15.42:6831","nonce":1633099631}]},"public_addr":"172.21.15.42:6827/1633099631","cluster_addr":"172.21.15.42:6829/1633099631","heartbeat_back_addr":"172.21.15.42:6833/1633099631","heartbeat_front_addr":"172.21.15.42:6831/1633099631","state":["exists","up"]},{"osd":4,"uuid":"32958af8-ae8a-4abc-95d9-7c08ac3f758e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":2353291645},{"type":"v1","addr":"172.21.15.169:6801","nonce":2353291645}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":2353291645},{"type":"v1","addr":"172.21.15.169:6803","nonce":2353291645}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":2353291645},{"type":"v1","addr":"172.21.15.169:6807","nonce":2353291645}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":2353291645},{"type":"v1","addr":"172.21.15.169:6805","nonce":2353291645}]},"public_addr":"172.21.15.169:6801/2353291645","cluster_addr":"172.21.15.169:6803/2353291645","heartbeat_back_addr":"172.21.15.169:6807/2353291645","heartbeat_front_addr":"172.21.15.169:6805/2353291645","state":["exists","up"]},{"osd":5,"uuid":"46d22890-4110-43a7-9c0d-7406495a9924","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":1262879371},{"type":"v1","addr":"172.21.15.169:6809","nonce":1262879371}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":1262879371},{"type":"v1","addr":"172.21.15.169:6811","nonce":1262879371}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":1262879371},{"type":"v1","addr":"172.21.15.169:6815","nonce":1262879371}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":1262879371},{"type":"v1","addr":"172.21.15.169:6813","nonce":1262879371}]},"public_addr":"172.21.15.169:6809/1262879371","cluster_addr":"172.21.15.169:6811/1262879371","heartbeat_back_addr":"172.21.15.169:6815/1262879371","heartbeat_front_addr":"172.21.15.169:6813/1262879371","state":["exists","up"]},{"osd":6,"uuid":"aca7c09b-7ae9-45b2-91d2-48413a58886e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":3799859416},{"type":"v1","addr":"172.21.15.169:6817","nonce":3799859416}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":3799859416},{"type":"v1","addr":"172.21.15.169:6819","nonce":3799859416}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6822","nonce":3799859416},{"type":"v1","addr":"172.21.15.169:6823","nonce":3799859416}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":3799859416},{"type":"v1","addr":"172.21.15.169:6821","nonce":3799859416}]},"public_addr":"172.21.15.169:6817/3799859416","cluster_addr":"172.21.15.169:6819/3799859416","heartbeat_back_addr":"172.21.15.169:6823/3799859416","heartbeat_front_addr":"172.21.15.169:6821/3799859416","state":["exists","up"]},{"osd":7,"uuid":"661346d4-14d3-446f-93cc-8fc8f6fd9470","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6824","nonce":4048323867},{"type":"v1","addr":"172.21.15.169:6825","nonce":4048323867}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6826","nonce":4048323867},{"type":"v1","addr":"172.21.15.169:6827","nonce":4048323867}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6830","nonce":4048323867},{"type":"v1","addr":"172.21.15.169:6831","nonce":4048323867}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6828","nonce":4048323867},{"type":"v1","addr":"172.21.15.169:6829","nonce":4048323867}]},"public_addr":"172.21.15.169:6825/4048323867","cluster_addr":"172.21.15.169:6827/4048323867","heartbeat_back_addr":"172.21.15.169:6831/4048323867","heartbeat_front_addr":"172.21.15.169:6829/4048323867","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:57:48.227447+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:58:12.105585+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:58:35.642777+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:58:59.867595+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:59:20.837579+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:59:43.638559+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:00:06.825493+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:00:30.029025+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.42:6800/2635164233":"2024-04-03T15:00:47.490451+0000","172.21.15.42:0/280964113":"2024-04-03T15:00:47.490451+0000","172.21.15.42:0/3168280765":"2024-04-03T14:56:30.584985+0000","172.21.15.42:0/2888334500":"2024-04-03T15:00:47.490451+0000","172.21.15.42:0/1443281804":"2024-04-03T14:56:30.584985+0000","172.21.15.42:0/2350866342":"2024-04-03T15:00:47.490451+0000","172.21.15.42:0/3792138265":"2024-04-03T15:00:47.490451+0000","172.21.15.42:6801/1586110338":"2024-04-03T14:56:30.584985+0000","172.21.15.42:6800/1586110338":"2024-04-03T14:56:30.584985+0000","172.21.15.42:6801/2635164233":"2024-04-03T15:00:47.490451+0000","172.21.15.42:0/3226246413":"2024-04-03T14:56:30.584985+0000","172.21.15.42:0/1870022407":"2024-04-03T14:56:08.269630+0000","172.21.15.42:0/3766045960":"2024-04-03T14:56:08.269630+0000","172.21.15.42:0/1412464921":"2024-04-03T14:56:08.269630+0000","172.21.15.42:6800/3054125886":"2024-04-03T14:56:08.269630+0000","172.21.15.42:6801/3054125886":"2024-04-03T14:56:08.269630+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T15:01:10.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:10 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3636955330' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:01:10.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:10 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3636955330' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:01:10.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:10 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3636955330' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:01:10.794 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:10 smithi042 bash[44285]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-02T15:01:11.108 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:10 smithi042 bash[44285]: Writing manifest to image destination 2024-04-02T15:01:11.132 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-02T15:01:11.132 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-02T15:01:11.434 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 podman[44285]: 2024-04-02 15:01:11.145107478 +0000 UTC m=+3.504405417 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-02T15:01:11.434 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 podman[44285]: 2024-04-02 15:01:11.260267118 +0000 UTC m=+3.619565054 container create d0a5ca877a2af0c19c65cf92f60a3702b55dc128d90867b307dc800a04088489 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T15:01:11.435 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 podman[44285]: 2024-04-02 15:01:11.44572155 +0000 UTC m=+3.805019499 container init d0a5ca877a2af0c19c65cf92f60a3702b55dc128d90867b307dc800a04088489 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T15:01:11.539 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:11.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:11 smithi169 ceph-mon[25801]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:11.768 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-mon[24805]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:11.768 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-mon[29473]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:11.769 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 podman[44285]: 2024-04-02 15:01:11.448397063 +0000 UTC m=+3.807695013 container start d0a5ca877a2af0c19c65cf92f60a3702b55dc128d90867b307dc800a04088489 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T15:01:11.769 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.452Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-02T15:01:11.769 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.452Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-02T15:01:11.769 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.452Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-02T15:01:11.769 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.452Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-02T15:01:11.769 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-02T15:01:11.769 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=arp 2024-04-02T15:01:11.769 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-02T15:01:11.769 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-02T15:01:11.769 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-02T15:01:11.769 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-02T15:01:11.770 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-02T15:01:11.770 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-02T15:01:11.770 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-02T15:01:11.770 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-02T15:01:11.770 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=edac 2024-04-02T15:01:11.770 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-02T15:01:11.770 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-02T15:01:11.770 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-02T15:01:11.771 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-02T15:01:11.771 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-02T15:01:11.771 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-02T15:01:11.771 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-02T15:01:11.771 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-02T15:01:11.771 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-02T15:01:11.771 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-02T15:01:11.771 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-02T15:01:11.771 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-02T15:01:11.771 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-02T15:01:11.771 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-02T15:01:11.771 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-02T15:01:11.771 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-02T15:01:11.771 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=os 2024-04-02T15:01:11.772 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-02T15:01:11.772 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-02T15:01:11.772 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-02T15:01:11.772 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-02T15:01:11.772 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-02T15:01:11.772 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-02T15:01:11.772 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=stat 2024-04-02T15:01:11.772 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-02T15:01:11.772 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-02T15:01:11.772 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-02T15:01:11.772 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=time 2024-04-02T15:01:11.772 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-02T15:01:11.772 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=uname 2024-04-02T15:01:11.772 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-02T15:01:11.773 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-02T15:01:11.773 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-02T15:01:11.773 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-02T15:01:11.773 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a[44490]: ts=2024-04-02T15:01:11.453Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-02T15:01:11.773 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 bash[44285]: d0a5ca877a2af0c19c65cf92f60a3702b55dc128d90867b307dc800a04088489 2024-04-02T15:01:11.773 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:01:11 smithi042 systemd[1]: Started Ceph node-exporter.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T15:01:12.029 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:01:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:01:11] "GET /metrics HTTP/1.1" 200 191508 "" "Prometheus/2.33.4" 2024-04-02T15:01:12.431 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:01:12 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:01:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:01:12.432 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:12 smithi169 systemd[1]: Starting Ceph node-exporter.b for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:01:12.666 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:12 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:12.667 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:12 smithi042 ceph-mon[24805]: Deploying daemon node-exporter.b on smithi169 2024-04-02T15:01:12.667 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:12 smithi042 ceph-mon[24805]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:12.667 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:12 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:12.667 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:12 smithi042 ceph-mon[29473]: Deploying daemon node-exporter.b on smithi169 2024-04-02T15:01:12.667 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:12 smithi042 ceph-mon[29473]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:12.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:12 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:12.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:12 smithi169 ceph-mon[25801]: Deploying daemon node-exporter.b on smithi169 2024-04-02T15:01:12.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:12 smithi169 ceph-mon[25801]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:12.728 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:12 smithi169 bash[41172]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-04-02T15:01:13.227 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:12 smithi169 bash[41172]: Getting image source signatures 2024-04-02T15:01:13.227 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:12 smithi169 bash[41172]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-04-02T15:01:13.227 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:12 smithi169 bash[41172]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-04-02T15:01:13.227 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:12 smithi169 bash[41172]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-02T15:01:13.319 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:01:13.319 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":60,"fsid":"fd27f2e2-f100-11ee-b647-cb9ed24678a4","created":"2024-04-02T14:55:40.720004+0000","modified":"2024-04-02T15:00:47.490511+0000","last_up_change":"2024-04-02T15:00:31.248602+0000","last_in_change":"2024-04-02T15:00:14.786180+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T14:58:39.487535+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"177da1b0-ec41-4bbe-8d6b-988afdd4cbbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6802","nonce":2441352487},{"type":"v1","addr":"172.21.15.42:6803","nonce":2441352487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6804","nonce":2441352487},{"type":"v1","addr":"172.21.15.42:6805","nonce":2441352487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6808","nonce":2441352487},{"type":"v1","addr":"172.21.15.42:6809","nonce":2441352487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6806","nonce":2441352487},{"type":"v1","addr":"172.21.15.42:6807","nonce":2441352487}]},"public_addr":"172.21.15.42:6803/2441352487","cluster_addr":"172.21.15.42:6805/2441352487","heartbeat_back_addr":"172.21.15.42:6809/2441352487","heartbeat_front_addr":"172.21.15.42:6807/2441352487","state":["exists","up"]},{"osd":1,"uuid":"aac29b7a-cf39-4dee-87e3-a905551d6230","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6810","nonce":146974053},{"type":"v1","addr":"172.21.15.42:6811","nonce":146974053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6812","nonce":146974053},{"type":"v1","addr":"172.21.15.42:6813","nonce":146974053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6816","nonce":146974053},{"type":"v1","addr":"172.21.15.42:6817","nonce":146974053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6814","nonce":146974053},{"type":"v1","addr":"172.21.15.42:6815","nonce":146974053}]},"public_addr":"172.21.15.42:6811/146974053","cluster_addr":"172.21.15.42:6813/146974053","heartbeat_back_addr":"172.21.15.42:6817/146974053","heartbeat_front_addr":"172.21.15.42:6815/146974053","state":["exists","up"]},{"osd":2,"uuid":"710ff1f2-d67f-405c-b48b-64e9fd8b6b68","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6818","nonce":744264554},{"type":"v1","addr":"172.21.15.42:6819","nonce":744264554}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6820","nonce":744264554},{"type":"v1","addr":"172.21.15.42:6821","nonce":744264554}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6824","nonce":744264554},{"type":"v1","addr":"172.21.15.42:6825","nonce":744264554}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6822","nonce":744264554},{"type":"v1","addr":"172.21.15.42:6823","nonce":744264554}]},"public_addr":"172.21.15.42:6819/744264554","cluster_addr":"172.21.15.42:6821/744264554","heartbeat_back_addr":"172.21.15.42:6825/744264554","heartbeat_front_addr":"172.21.15.42:6823/744264554","state":["exists","up"]},{"osd":3,"uuid":"27d9b9d7-e1f8-4be8-9da8-13dd4da8dad8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6826","nonce":1633099631},{"type":"v1","addr":"172.21.15.42:6827","nonce":1633099631}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6828","nonce":1633099631},{"type":"v1","addr":"172.21.15.42:6829","nonce":1633099631}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6832","nonce":1633099631},{"type":"v1","addr":"172.21.15.42:6833","nonce":1633099631}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6830","nonce":1633099631},{"type":"v1","addr":"172.21.15.42:6831","nonce":1633099631}]},"public_addr":"172.21.15.42:6827/1633099631","cluster_addr":"172.21.15.42:6829/1633099631","heartbeat_back_addr":"172.21.15.42:6833/1633099631","heartbeat_front_addr":"172.21.15.42:6831/1633099631","state":["exists","up"]},{"osd":4,"uuid":"32958af8-ae8a-4abc-95d9-7c08ac3f758e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":2353291645},{"type":"v1","addr":"172.21.15.169:6801","nonce":2353291645}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":2353291645},{"type":"v1","addr":"172.21.15.169:6803","nonce":2353291645}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":2353291645},{"type":"v1","addr":"172.21.15.169:6807","nonce":2353291645}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":2353291645},{"type":"v1","addr":"172.21.15.169:6805","nonce":2353291645}]},"public_addr":"172.21.15.169:6801/2353291645","cluster_addr":"172.21.15.169:6803/2353291645","heartbeat_back_addr":"172.21.15.169:6807/2353291645","heartbeat_front_addr":"172.21.15.169:6805/2353291645","state":["exists","up"]},{"osd":5,"uuid":"46d22890-4110-43a7-9c0d-7406495a9924","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":1262879371},{"type":"v1","addr":"172.21.15.169:6809","nonce":1262879371}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":1262879371},{"type":"v1","addr":"172.21.15.169:6811","nonce":1262879371}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":1262879371},{"type":"v1","addr":"172.21.15.169:6815","nonce":1262879371}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":1262879371},{"type":"v1","addr":"172.21.15.169:6813","nonce":1262879371}]},"public_addr":"172.21.15.169:6809/1262879371","cluster_addr":"172.21.15.169:6811/1262879371","heartbeat_back_addr":"172.21.15.169:6815/1262879371","heartbeat_front_addr":"172.21.15.169:6813/1262879371","state":["exists","up"]},{"osd":6,"uuid":"aca7c09b-7ae9-45b2-91d2-48413a58886e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":3799859416},{"type":"v1","addr":"172.21.15.169:6817","nonce":3799859416}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":3799859416},{"type":"v1","addr":"172.21.15.169:6819","nonce":3799859416}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6822","nonce":3799859416},{"type":"v1","addr":"172.21.15.169:6823","nonce":3799859416}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":3799859416},{"type":"v1","addr":"172.21.15.169:6821","nonce":3799859416}]},"public_addr":"172.21.15.169:6817/3799859416","cluster_addr":"172.21.15.169:6819/3799859416","heartbeat_back_addr":"172.21.15.169:6823/3799859416","heartbeat_front_addr":"172.21.15.169:6821/3799859416","state":["exists","up"]},{"osd":7,"uuid":"661346d4-14d3-446f-93cc-8fc8f6fd9470","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6824","nonce":4048323867},{"type":"v1","addr":"172.21.15.169:6825","nonce":4048323867}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6826","nonce":4048323867},{"type":"v1","addr":"172.21.15.169:6827","nonce":4048323867}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6830","nonce":4048323867},{"type":"v1","addr":"172.21.15.169:6831","nonce":4048323867}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6828","nonce":4048323867},{"type":"v1","addr":"172.21.15.169:6829","nonce":4048323867}]},"public_addr":"172.21.15.169:6825/4048323867","cluster_addr":"172.21.15.169:6827/4048323867","heartbeat_back_addr":"172.21.15.169:6831/4048323867","heartbeat_front_addr":"172.21.15.169:6829/4048323867","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:57:48.227447+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:58:12.105585+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:58:35.642777+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:58:59.867595+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:59:20.837579+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:59:43.638559+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:00:06.825493+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:00:30.029025+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.42:6800/2635164233":"2024-04-03T15:00:47.490451+0000","172.21.15.42:0/280964113":"2024-04-03T15:00:47.490451+0000","172.21.15.42:0/3168280765":"2024-04-03T14:56:30.584985+0000","172.21.15.42:0/2888334500":"2024-04-03T15:00:47.490451+0000","172.21.15.42:0/1443281804":"2024-04-03T14:56:30.584985+0000","172.21.15.42:0/2350866342":"2024-04-03T15:00:47.490451+0000","172.21.15.42:0/3792138265":"2024-04-03T15:00:47.490451+0000","172.21.15.42:6801/1586110338":"2024-04-03T14:56:30.584985+0000","172.21.15.42:6800/1586110338":"2024-04-03T14:56:30.584985+0000","172.21.15.42:6801/2635164233":"2024-04-03T15:00:47.490451+0000","172.21.15.42:0/3226246413":"2024-04-03T14:56:30.584985+0000","172.21.15.42:0/1870022407":"2024-04-03T14:56:08.269630+0000","172.21.15.42:0/3766045960":"2024-04-03T14:56:08.269630+0000","172.21.15.42:0/1412464921":"2024-04-03T14:56:08.269630+0000","172.21.15.42:6800/3054125886":"2024-04-03T14:56:08.269630+0000","172.21.15.42:6801/3054125886":"2024-04-03T14:56:08.269630+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T15:01:13.957 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:13 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/147201607' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:01:13.957 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:13 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/147201607' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:01:13.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:13 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/147201607' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:01:13.983 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-02T15:01:13.983 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-02T15:01:13.983 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-02T15:01:13.984 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-02T15:01:13.984 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-02T15:01:13.985 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-02T15:01:13.985 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-04-02T15:01:13.986 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-04-02T15:01:14.407 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:14.410 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:14.410 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:14.411 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:14.412 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:14.469 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:14.469 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:14.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:14 smithi042 ceph-mon[24805]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:14.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:14 smithi042 ceph-mon[29473]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:14.891 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:14 smithi169 ceph-mon[25801]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:14.891 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:14 smithi169 bash[41172]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-02T15:01:15.161 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:14 smithi169 bash[41172]: Writing manifest to image destination 2024-04-02T15:01:15.423 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 podman[41172]: 2024-04-02 15:01:15.167880294 +0000 UTC m=+2.818065476 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-02T15:01:15.423 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 podman[41172]: 2024-04-02 15:01:15.281760185 +0000 UTC m=+2.931945346 container create dbb30e649ccfc88ac848190e9f5fadeb9261894ee68a7139a310a309cbd5cedb (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T15:01:15.730 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 podman[41172]: 2024-04-02 15:01:15.467519687 +0000 UTC m=+3.117704863 container init dbb30e649ccfc88ac848190e9f5fadeb9261894ee68a7139a310a309cbd5cedb (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T15:01:15.730 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 podman[41172]: 2024-04-02 15:01:15.470403164 +0000 UTC m=+3.120588328 container start dbb30e649ccfc88ac848190e9f5fadeb9261894ee68a7139a310a309cbd5cedb (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T15:01:15.730 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.474Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-02T15:01:15.730 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.474Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-02T15:01:15.731 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.474Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-02T15:01:15.731 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.474Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-02T15:01:15.731 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-02T15:01:15.731 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=arp 2024-04-02T15:01:15.731 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-02T15:01:15.731 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-02T15:01:15.731 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-02T15:01:15.731 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-02T15:01:15.731 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-02T15:01:15.731 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-02T15:01:15.731 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-02T15:01:15.731 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-02T15:01:15.731 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=edac 2024-04-02T15:01:15.731 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-02T15:01:15.732 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-02T15:01:15.732 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-02T15:01:15.732 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-02T15:01:15.732 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-02T15:01:15.732 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-02T15:01:15.732 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-02T15:01:15.732 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-02T15:01:15.732 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-02T15:01:15.732 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-02T15:01:15.732 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-02T15:01:15.732 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-02T15:01:15.732 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-02T15:01:15.733 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-02T15:01:15.733 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-02T15:01:15.733 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-02T15:01:15.733 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=os 2024-04-02T15:01:15.733 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-02T15:01:15.733 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-02T15:01:15.733 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-02T15:01:15.733 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-02T15:01:15.733 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-02T15:01:15.733 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-02T15:01:15.733 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=stat 2024-04-02T15:01:15.733 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-02T15:01:15.733 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-02T15:01:15.734 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-02T15:01:15.734 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=time 2024-04-02T15:01:15.734 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-02T15:01:15.734 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=uname 2024-04-02T15:01:15.734 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-02T15:01:15.734 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-02T15:01:15.734 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-02T15:01:15.734 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-02T15:01:15.734 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b[41232]: ts=2024-04-02T15:01:15.475Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-02T15:01:15.734 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 bash[41172]: dbb30e649ccfc88ac848190e9f5fadeb9261894ee68a7139a310a309cbd5cedb 2024-04-02T15:01:15.734 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:01:15 smithi169 systemd[1]: Started Ceph node-exporter.b for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T15:01:16.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:16 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:16.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:16 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:01:16.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:16 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:01:16.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:16 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:01:16.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:16 smithi042 ceph-mon[24805]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:16.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:16 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:16.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:16 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:01:16.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:16 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:01:16.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:16 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:01:16.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:16 smithi042 ceph-mon[29473]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:16.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:16 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:16.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:16 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:01:16.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:16 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:01:16.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:16 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:01:16.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:16 smithi169 ceph-mon[25801]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:19.402 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:19 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:19.402 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:19 smithi042 ceph-mon[24805]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:19.402 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:19 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:19.403 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:19 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:19.403 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:19 smithi042 ceph-mon[29473]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:19.403 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:19 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:19.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:19 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:19.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:19 smithi169 ceph-mon[25801]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:19.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:19 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:19.768 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:21.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:21 smithi042 ceph-mon[24805]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:21.293 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:21 smithi042 ceph-mon[29473]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:21.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:21 smithi169 ceph-mon[25801]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:22.109 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:01:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:01:21] "GET /metrics HTTP/1.1" 200 191499 "" "Prometheus/2.33.4" 2024-04-02T15:01:22.477 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:01:22 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:01:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:01:23.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:23 smithi169 ceph-mon[25801]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:23.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:23 smithi042 ceph-mon[24805]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:23.797 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:23 smithi042 ceph-mon[29473]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:24.324 INFO:teuthology.orchestra.run.smithi042.stdout:167503724565 2024-04-02T15:01:24.325 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-02T15:01:25.647 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:25 smithi042 ceph-mon[24805]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:25.647 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:25 smithi042 ceph-mon[29473]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:25.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:25 smithi169 ceph-mon[25801]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:26.146 INFO:teuthology.orchestra.run.smithi042.stdout:111669149725 2024-04-02T15:01:26.147 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-02T15:01:27.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:27 smithi169 ceph-mon[25801]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:27.750 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:27 smithi042 ceph-mon[29473]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:27.751 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:27 smithi042 ceph-mon[24805]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:28.329 INFO:teuthology.orchestra.run.smithi042.stdout:236223201293 2024-04-02T15:01:28.330 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-04-02T15:01:28.973 INFO:teuthology.orchestra.run.smithi042.stdout:201863462930 2024-04-02T15:01:28.974 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-04-02T15:01:29.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:29 smithi042 ceph-mon[24805]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:29.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:29 smithi042 ceph-mon[29473]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:29.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:29 smithi169 ceph-mon[25801]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:30.059 INFO:teuthology.orchestra.run.smithi042.stdout:77309411363 2024-04-02T15:01:30.060 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-02T15:01:30.458 INFO:teuthology.orchestra.run.smithi042.stdout:34359738412 2024-04-02T15:01:30.459 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-02T15:01:30.691 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:30.692 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:31.215 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:31.222 INFO:teuthology.orchestra.run.smithi042.stdout:55834574888 2024-04-02T15:01:31.222 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-02T15:01:31.252 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:31.485 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:31 smithi042 ceph-mon[24805]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:31.486 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:31 smithi042 ceph-mon[29473]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:31.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:31 smithi169 ceph-mon[25801]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:32.293 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:01:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:01:31] "GET /metrics HTTP/1.1" 200 191499 "" "Prometheus/2.33.4" 2024-04-02T15:01:32.477 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:01:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:01:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:01:33.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:33 smithi042 ceph-mon[24805]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:33.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:33 smithi042 ceph-mon[29473]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:33.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:33 smithi169 ceph-mon[25801]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:35.070 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:35.091 INFO:teuthology.orchestra.run.smithi042.stdout:133143986202 2024-04-02T15:01:35.092 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-02T15:01:35.734 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:35 smithi042 ceph-mon[29473]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:35.734 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:35 smithi042 ceph-mon[24805]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:35.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:35 smithi169 ceph-mon[25801]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:36.306 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:36.306 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:37.737 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:37 smithi042 ceph-mon[24805]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:37.737 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:37 smithi042 ceph-mon[29473]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:37.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:37 smithi169 ceph-mon[25801]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:39.468 INFO:teuthology.orchestra.run.smithi042.stdout:167503724568 2024-04-02T15:01:39.633 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:39 smithi042 ceph-mon[24805]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:39.633 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:39 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1030884080' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-02T15:01:39.633 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:39 smithi042 ceph-mon[29473]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:39.633 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:39 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1030884080' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-02T15:01:39.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:39 smithi169 ceph-mon[25801]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:39.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:39 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1030884080' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-02T15:01:40.256 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724565 got 167503724568 for osd.5 2024-04-02T15:01:40.256 DEBUG:teuthology.parallel:result is None 2024-04-02T15:01:41.480 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:41 smithi042 ceph-mon[24805]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:41.480 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:41 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/2942035049' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-02T15:01:41.480 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:41 smithi042 ceph-mon[29473]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:41.481 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:41 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/2942035049' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-02T15:01:41.546 INFO:teuthology.orchestra.run.smithi042.stdout:111669149729 2024-04-02T15:01:41.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:41 smithi169 ceph-mon[25801]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:41.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:41 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/2942035049' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-02T15:01:42.229 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:01:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:01:41] "GET /metrics HTTP/1.1" 200 191486 "" "Prometheus/2.33.4" 2024-04-02T15:01:42.253 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149725 got 111669149729 for osd.3 2024-04-02T15:01:42.253 DEBUG:teuthology.parallel:result is None 2024-04-02T15:01:42.343 INFO:teuthology.orchestra.run.smithi042.stdout:236223201295 2024-04-02T15:01:42.477 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:01:42 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:01:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:01:42.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:42 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/735954067' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-02T15:01:42.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:42 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:42.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:42 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:42.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:42 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/735954067' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-02T15:01:42.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:42 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:42.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:42 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:42.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:42 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/735954067' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-02T15:01:42.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:42 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:42.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:42 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:42.841 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201293 got 236223201295 for osd.7 2024-04-02T15:01:42.841 DEBUG:teuthology.parallel:result is None 2024-04-02T15:01:43.073 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:43.279 INFO:teuthology.orchestra.run.smithi042.stdout:201863462933 2024-04-02T15:01:43.505 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:43 smithi042 ceph-mon[24805]: Deploying daemon alertmanager.a on smithi042 2024-04-02T15:01:43.505 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:43 smithi042 ceph-mon[24805]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:43.505 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:43 smithi042 ceph-mon[29473]: Deploying daemon alertmanager.a on smithi042 2024-04-02T15:01:43.505 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:43 smithi042 ceph-mon[29473]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:43.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:43 smithi169 ceph-mon[25801]: Deploying daemon alertmanager.a on smithi042 2024-04-02T15:01:43.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:43 smithi169 ceph-mon[25801]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:44.495 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:44 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1242098614' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-02T15:01:44.495 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:44 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1242098614' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-02T15:01:44.628 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:44 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1242098614' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-02T15:01:45.193 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462930 got 201863462933 for osd.6 2024-04-02T15:01:45.193 DEBUG:teuthology.parallel:result is None 2024-04-02T15:01:45.418 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:45 smithi042 ceph-mon[24805]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:45.418 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:45 smithi042 ceph-mon[29473]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:45.531 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:45 smithi169 ceph-mon[25801]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:46.301 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:46 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/2695333274' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-02T15:01:46.302 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:46 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/2695333274' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-02T15:01:46.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:46 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/2695333274' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-02T15:01:46.919 INFO:teuthology.orchestra.run.smithi042.stdout:55834574891 2024-04-02T15:01:46.922 INFO:teuthology.orchestra.run.smithi042.stdout:34359738415 2024-04-02T15:01:47.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:47 smithi042 ceph-mon[24805]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:47.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:47 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/755134727' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-02T15:01:47.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:47 smithi042 ceph-mon[29473]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:47.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:47 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/755134727' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-02T15:01:47.702 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738412 got 34359738415 for osd.0 2024-04-02T15:01:47.702 DEBUG:teuthology.parallel:result is None 2024-04-02T15:01:47.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:47 smithi169 ceph-mon[25801]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:47.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:47 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/755134727' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-02T15:01:48.311 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574888 got 55834574891 for osd.1 2024-04-02T15:01:48.312 DEBUG:teuthology.parallel:result is None 2024-04-02T15:01:48.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:48 smithi042 ceph-mon[24805]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:48.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:01:48.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:01:48.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:01:48.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:01:48.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:48 smithi042 ceph-mon[29473]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:48.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:01:48.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:01:48.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:01:48.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:01:48.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:48 smithi169 ceph-mon[25801]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:48.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:01:48.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:01:48.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:01:48.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:01:49.747 INFO:teuthology.orchestra.run.smithi042.stdout:133143986206 2024-04-02T15:01:50.031 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:49 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3941201686' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-02T15:01:50.032 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:49 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3941201686' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-02T15:01:50.197 INFO:teuthology.orchestra.run.smithi042.stdout:77309411367 2024-04-02T15:01:50.227 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:49 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3941201686' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-02T15:01:51.227 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:50 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/2698104326' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-02T15:01:51.227 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:50 smithi169 ceph-mon[25801]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:51.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:50 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/2698104326' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-02T15:01:51.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:50 smithi042 ceph-mon[24805]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:51.293 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:50 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/2698104326' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-02T15:01:51.293 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:50 smithi042 ceph-mon[29473]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:51.329 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411363 got 77309411367 for osd.2 2024-04-02T15:01:51.329 DEBUG:teuthology.parallel:result is None 2024-04-02T15:01:51.722 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986202 got 133143986206 for osd.4 2024-04-02T15:01:51.722 DEBUG:teuthology.parallel:result is None 2024-04-02T15:01:51.722 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-02T15:01:51.722 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-02T15:01:52.043 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:01:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:01:51] "GET /metrics HTTP/1.1" 200 191483 "" "Prometheus/2.33.4" 2024-04-02T15:01:52.477 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:01:52 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:01:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:01:53.426 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:53.514 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:53 smithi042 ceph-mon[29473]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:53.514 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:53 smithi042 ceph-mon[24805]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:53.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:53 smithi169 ceph-mon[25801]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:55.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:55 smithi042 ceph-mon[24805]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:55.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:55 smithi042 ceph-mon[29473]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:55.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:55 smithi169 ceph-mon[25801]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:56.401 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:01:56.403 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-04-02T15:01:57.361 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":36,"stamp":"2024-04-02T15:01:54.436108+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48900,"kb_used_data":6148,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912316,"statfs":{"total":767960285184,"available":767910211584,"internally_reserved":0,"allocated":6295552,"data_stored":3509853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002409"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":262,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-02T15:00:48.506836+0000","last_change":"2024-04-02T15:00:35.281940+0000","last_active":"2024-04-02T15:00:48.506836+0000","last_peered":"2024-04-02T15:00:48.506836+0000","last_clean":"2024-04-02T15:00:48.506836+0000","last_became_active":"2024-04-02T15:00:35.281453+0000","last_became_peered":"2024-04-02T15:00:35.281453+0000","last_unstale":"2024-04-02T15:00:48.506836+0000","last_undegraded":"2024-04-02T15:00:48.506836+0000","last_fullsized":"2024-04-02T15:00:48.506836+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-02T14:58:39.570564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-02T14:58:39.570564+0000","last_clean_scrub_stamp":"2024-04-02T14:58:39.570564+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-03T18:55:42.169689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201298,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.52600000000000002}]},{"osd":1,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.42799999999999999}]},{"osd":2,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.41899999999999998}]},{"osd":3,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.441},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.54100000000000004}]},{"osd":4,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.55500000000000005}]},{"osd":5,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.5},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.59399999999999997}]},{"osd":6,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.72199999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462935,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6156,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738996,"statfs":{"total":95995035648,"available":95988731904,"internally_reserved":0,"allocated":1118208,"data_stored":769212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:01:09 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.56399999999999995}]},{"osd":1,"last update":"Tue Apr 2 15:01:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.55000000000000004}]},{"osd":2,"last update":"Tue Apr 2 15:01:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.60899999999999999}]},{"osd":3,"last update":"Tue Apr 2 15:01:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.51800000000000002}]},{"osd":4,"last update":"Tue Apr 2 15:01:09 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.65500000000000003}]},{"osd":5,"last update":"Tue Apr 2 15:01:09 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.502},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.57599999999999996}]},{"osd":7,"last update":"Tue Apr 2 15:01:33 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.624},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.63600000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574893,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":240342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:01:20 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.35199999999999998,"15min":0.33400000000000002},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.57599999999999996,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.379},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.373,"15min":0.35199999999999998},"min":{"1min":0.26600000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.26700000000000002}]},{"osd":2,"last update":"Tue Apr 2 15:01:40 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.28899999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.55800000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.41999999999999998,"15min":0.41399999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60599999999999998,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.30099999999999999}]},{"osd":3,"last update":"Tue Apr 2 15:01:05 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.40699999999999997,"15min":0.40400000000000003},"min":{"1min":0.308,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.317},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.46999999999999997,"15min":0.47199999999999998},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.58499999999999996,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.36699999999999999}]},{"osd":4,"last update":"Tue Apr 2 15:01:31 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.61099999999999999,"15min":0.61399999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.81999999999999995,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.347,"15min":1.4690000000000001},"min":{"1min":0.44700000000000001,"5min":0.439,"15min":0.439},"max":{"1min":0.84299999999999997,"5min":13.972,"15min":13.972},"last":0.56999999999999995}]},{"osd":5,"last update":"Tue Apr 2 15:01:49 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.63300000000000001,"15min":0.63400000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.66000000000000003,"15min":0.66300000000000003},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.54900000000000004}]},{"osd":6,"last update":"Tue Apr 2 15:01:13 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.504},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.58699999999999997}]},{"osd":7,"last update":"Tue Apr 2 15:01:31 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.59999999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738417,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Apr 2 15:01:18 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.377,"15min":0.36299999999999999},"min":{"1min":0.25800000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.60399999999999998,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.39400000000000002},"min":{"1min":0.28799999999999998,"5min":0.217,"15min":0.217},"max":{"1min":0.71899999999999997,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.27800000000000002}]},{"osd":2,"last update":"Tue Apr 2 15:01:45 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.41199999999999998,"15min":0.40400000000000003},"min":{"1min":0.28499999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.625,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.377,"5min":0.41099999999999998,"15min":0.41399999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.58799999999999997,"5min":0.627,"15min":0.627},"last":0.26600000000000001}]},{"osd":3,"last update":"Tue Apr 2 15:01:12 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.45100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.317},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.49399999999999999,"15min":0.499},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.68100000000000005,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.376}]},{"osd":4,"last update":"Tue Apr 2 15:01:27 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.623,"15min":0.625},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.60999999999999999,"15min":0.61399999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.435}]},{"osd":5,"last update":"Tue Apr 2 15:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56899999999999995,"15min":0.57099999999999995},"min":{"1min":0.39700000000000002,"5min":0.377,"15min":0.377},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.58699999999999997,"15min":0.58999999999999997},"min":{"1min":0.42799999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.52600000000000002}]},{"osd":6,"last update":"Tue Apr 2 15:01:12 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.47799999999999998}]},{"osd":7,"last update":"Tue Apr 2 15:01:35 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.62}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6276,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738876,"statfs":{"total":95995035648,"available":95988609024,"internally_reserved":0,"allocated":585728,"data_stored":239996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.36899999999999999,"15min":0.35999999999999999},"min":{"1min":0.25900000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.53000000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.39300000000000002,"15min":0.38100000000000001},"min":{"1min":0.22,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.48399999999999999}]},{"osd":1,"last update":"Tue Apr 2 15:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.38400000000000001,"15min":0.378},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.58399999999999996,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.41799999999999998,"15min":0.40799999999999997},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.58599999999999997,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.44700000000000001}]},{"osd":3,"last update":"Tue Apr 2 15:01:04 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.46300000000000002,"15min":0.46100000000000002},"min":{"1min":0.34399999999999997,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.67500000000000004,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.435,"15min":0.43099999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.66500000000000004,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.47499999999999998}]},{"osd":4,"last update":"Tue Apr 2 15:01:28 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.60999999999999999,"15min":0.61599999999999999},"min":{"1min":0.45100000000000001,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.628,"15min":0.63200000000000001},"min":{"1min":0.45800000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.89000000000000001,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.85299999999999998}]},{"osd":5,"last update":"Tue Apr 2 15:01:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.60299999999999998,"15min":0.60899999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.78200000000000003,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.60199999999999998,"15min":0.60899999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.81899999999999995,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.81899999999999995}]},{"osd":6,"last update":"Tue Apr 2 15:01:10 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.75600000000000001}]},{"osd":7,"last update":"Tue Apr 2 15:01:34 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.78600000000000003}]}]},{"osd":3,"up_from":26,"seq":111669149731,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:01:10 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.40200000000000002,"15min":0.40000000000000002},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.433,"5min":0.42499999999999999,"15min":0.42299999999999999},"min":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.64000000000000001,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.38500000000000001}]},{"osd":1,"last update":"Tue Apr 2 15:01:10 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.438,"15min":0.441},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.44600000000000001,"15min":0.44400000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.307}]},{"osd":2,"last update":"Tue Apr 2 15:01:10 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.47099999999999997,"15min":0.47299999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.56299999999999994,"5min":0.62,"15min":0.62},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45100000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.56399999999999995,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.37}]},{"osd":4,"last update":"Tue Apr 2 15:01:23 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.57999999999999996,"15min":0.57899999999999996},"min":{"1min":0.39600000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.075,"5min":1.075,"15min":1.075},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59499999999999997,"15min":0.59699999999999998},"min":{"1min":0.42999999999999999,"5min":0.379,"15min":0.379},"max":{"1min":0.73199999999999998,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.66700000000000004}]},{"osd":5,"last update":"Tue Apr 2 15:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59999999999999998,"15min":0.60199999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.623,"5min":0.62,"15min":0.62},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.79700000000000004,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.753}]},{"osd":6,"last update":"Tue Apr 2 15:01:10 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.56999999999999995}]},{"osd":7,"last update":"Tue Apr 2 15:01:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.74199999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986207,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:01:23 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.58699999999999997,"15min":0.59899999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.66000000000000003,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.60699999999999998,"15min":0.61699999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.68300000000000005,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.51000000000000001}]},{"osd":1,"last update":"Tue Apr 2 15:01:23 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.54700000000000004,"15min":0.55700000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.60899999999999999,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.498,"5min":0.55800000000000005,"15min":0.56799999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.63900000000000001,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.48199999999999998}]},{"osd":2,"last update":"Tue Apr 2 15:01:23 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.56699999999999995,"15min":0.57899999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.74399999999999999,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.441},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.55900000000000005,"15min":0.56999999999999995},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.63100000000000001,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.40699999999999997}]},{"osd":3,"last update":"Tue Apr 2 15:01:23 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.58499999999999996,"15min":0.60099999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.68600000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.55500000000000005,"15min":0.56100000000000005},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.64800000000000002,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.374}]},{"osd":5,"last update":"Tue Apr 2 15:00:47 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.37,"5min":0.37,"15min":0.37},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.39300000000000002}]},{"osd":6,"last update":"Tue Apr 2 15:01:12 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.35599999999999998}]},{"osd":7,"last update":"Tue Apr 2 15:01:33 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.46500000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724571,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":240019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:00:46 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.70999999999999996}]},{"osd":1,"last update":"Tue Apr 2 15:00:46 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.73099999999999998}]},{"osd":2,"last update":"Tue Apr 2 15:00:46 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.65100000000000002}]},{"osd":3,"last update":"Tue Apr 2 15:00:46 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.58999999999999997}]},{"osd":4,"last update":"Tue Apr 2 15:00:46 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.66500000000000004}]},{"osd":6,"last update":"Tue Apr 2 15:01:10 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.754}]},{"osd":7,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.748},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.77300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-02T15:01:57.364 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-02T15:01:57.586 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:01:57.705 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:57 smithi042 ceph-mon[24805]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T15:01:57.706 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:57 smithi042 ceph-mon[24805]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:57.706 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:57 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:57.706 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:57 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:57.706 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:57 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:57.706 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:57 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T15:01:57.706 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:57 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:57.706 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:57 smithi042 ceph-mon[29473]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T15:01:57.706 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:57 smithi042 ceph-mon[29473]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:57.706 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:57 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:57.707 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:57 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:57.707 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:57 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:57.707 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:57 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T15:01:57.707 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:57 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:57.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:57 smithi169 ceph-mon[25801]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T15:01:57.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:57 smithi169 ceph-mon[25801]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:57.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:57 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:57.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:57 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:57.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:57 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:57.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:57 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T15:01:57.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:57 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:58.645 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:58 smithi042 ceph-mon[24805]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T15:01:58.645 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:58 smithi042 ceph-mon[24805]: Deploying daemon grafana.a on smithi169 2024-04-02T15:01:58.645 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:58 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:58.645 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:58 smithi042 ceph-mon[29473]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T15:01:58.645 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:58 smithi042 ceph-mon[29473]: Deploying daemon grafana.a on smithi169 2024-04-02T15:01:58.645 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:58 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:58.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:58 smithi169 ceph-mon[25801]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T15:01:58.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:58 smithi169 ceph-mon[25801]: Deploying daemon grafana.a on smithi169 2024-04-02T15:01:58.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:58 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:01:59.301 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:01:59.302 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-04-02T15:01:59.634 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:01:59 smithi042 ceph-mon[24805]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:59.634 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:01:59 smithi042 ceph-mon[29473]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:01:59.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:01:59 smithi169 ceph-mon[25801]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:00.021 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2024-04-02T15:01:58.436756+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48900,"kb_used_data":6148,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912316,"statfs":{"total":767960285184,"available":767910211584,"internally_reserved":0,"allocated":6295552,"data_stored":3509853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":262,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-02T15:00:48.506836+0000","last_change":"2024-04-02T15:00:35.281940+0000","last_active":"2024-04-02T15:00:48.506836+0000","last_peered":"2024-04-02T15:00:48.506836+0000","last_clean":"2024-04-02T15:00:48.506836+0000","last_became_active":"2024-04-02T15:00:35.281453+0000","last_became_peered":"2024-04-02T15:00:35.281453+0000","last_unstale":"2024-04-02T15:00:48.506836+0000","last_undegraded":"2024-04-02T15:00:48.506836+0000","last_fullsized":"2024-04-02T15:00:48.506836+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-02T14:58:39.570564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-02T14:58:39.570564+0000","last_clean_scrub_stamp":"2024-04-02T14:58:39.570564+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-03T18:55:42.169689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.59799999999999998}]},{"osd":1,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.55700000000000005}]},{"osd":2,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.53100000000000003}]},{"osd":3,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.621}]},{"osd":4,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.502}]},{"osd":5,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.64800000000000002}]},{"osd":6,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.63800000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462936,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6156,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738996,"statfs":{"total":95995035648,"available":95988731904,"internally_reserved":0,"allocated":1118208,"data_stored":769212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:01:09 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.63}]},{"osd":1,"last update":"Tue Apr 2 15:01:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.73199999999999998}]},{"osd":2,"last update":"Tue Apr 2 15:01:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.75}]},{"osd":3,"last update":"Tue Apr 2 15:01:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.70099999999999996}]},{"osd":4,"last update":"Tue Apr 2 15:01:09 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.51100000000000001}]},{"osd":5,"last update":"Tue Apr 2 15:01:09 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.59799999999999998}]},{"osd":7,"last update":"Tue Apr 2 15:01:33 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.57299999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":240342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:01:20 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.35199999999999998,"15min":0.33400000000000002},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.57599999999999996,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.373,"15min":0.35199999999999998},"min":{"1min":0.26600000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.36199999999999999}]},{"osd":2,"last update":"Tue Apr 2 15:01:40 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.28899999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.55800000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.41999999999999998,"15min":0.41399999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60599999999999998,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.42199999999999999}]},{"osd":3,"last update":"Tue Apr 2 15:01:05 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.40699999999999997,"15min":0.40400000000000003},"min":{"1min":0.308,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.46999999999999997,"15min":0.47199999999999998},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.58499999999999996,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.48699999999999999}]},{"osd":4,"last update":"Tue Apr 2 15:01:31 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.61099999999999999,"15min":0.61399999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.81999999999999995,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.501},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.347,"15min":1.4690000000000001},"min":{"1min":0.44700000000000001,"5min":0.439,"15min":0.439},"max":{"1min":0.84299999999999997,"5min":13.972,"15min":13.972},"last":0.59499999999999997}]},{"osd":5,"last update":"Tue Apr 2 15:01:49 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.63300000000000001,"15min":0.63400000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.438},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.66000000000000003,"15min":0.66300000000000003},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.57399999999999995}]},{"osd":6,"last update":"Tue Apr 2 15:01:13 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.56200000000000006}]},{"osd":7,"last update":"Tue Apr 2 15:01:31 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.622},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.65800000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738418,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Apr 2 15:01:18 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.377,"15min":0.36299999999999999},"min":{"1min":0.25800000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.60399999999999998,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.24299999999999999},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.39400000000000002},"min":{"1min":0.28799999999999998,"5min":0.217,"15min":0.217},"max":{"1min":0.71899999999999997,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.29199999999999998}]},{"osd":2,"last update":"Tue Apr 2 15:01:45 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.41199999999999998,"15min":0.40400000000000003},"min":{"1min":0.28499999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.625,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.377,"5min":0.41099999999999998,"15min":0.41399999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.58799999999999997,"5min":0.627,"15min":0.627},"last":0.35899999999999999}]},{"osd":3,"last update":"Tue Apr 2 15:01:12 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.45100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.49399999999999999,"15min":0.499},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.68100000000000005,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.315}]},{"osd":4,"last update":"Tue Apr 2 15:01:27 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.623,"15min":0.625},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.60999999999999999,"15min":0.61399999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.55400000000000005}]},{"osd":5,"last update":"Tue Apr 2 15:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56899999999999995,"15min":0.57099999999999995},"min":{"1min":0.39700000000000002,"5min":0.377,"15min":0.377},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.58699999999999997,"15min":0.58999999999999997},"min":{"1min":0.42799999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.45000000000000001}]},{"osd":6,"last update":"Tue Apr 2 15:01:12 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.42899999999999999}]},{"osd":7,"last update":"Tue Apr 2 15:01:35 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.48399999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411369,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6276,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738876,"statfs":{"total":95995035648,"available":95988609024,"internally_reserved":0,"allocated":585728,"data_stored":239996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.36899999999999999,"15min":0.35999999999999999},"min":{"1min":0.25900000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.53000000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.39300000000000002,"15min":0.38100000000000001},"min":{"1min":0.22,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.30199999999999999}]},{"osd":1,"last update":"Tue Apr 2 15:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.38400000000000001,"15min":0.378},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.58399999999999996,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.214},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.41799999999999998,"15min":0.40799999999999997},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.58599999999999997,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.252}]},{"osd":3,"last update":"Tue Apr 2 15:01:04 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.46300000000000002,"15min":0.46100000000000002},"min":{"1min":0.34399999999999997,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.67500000000000004,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.247},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.435,"15min":0.43099999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.66500000000000004,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.27800000000000002}]},{"osd":4,"last update":"Tue Apr 2 15:01:28 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.60999999999999999,"15min":0.61599999999999999},"min":{"1min":0.45100000000000001,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.628,"15min":0.63200000000000001},"min":{"1min":0.45800000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.89000000000000001,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.54500000000000004}]},{"osd":5,"last update":"Tue Apr 2 15:01:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.60299999999999998,"15min":0.60899999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.78200000000000003,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.60199999999999998,"15min":0.60899999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.81899999999999995,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.48599999999999999}]},{"osd":6,"last update":"Tue Apr 2 15:01:10 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.56499999999999995}]},{"osd":7,"last update":"Tue Apr 2 15:01:34 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.51700000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149732,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:01:10 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.40200000000000002,"15min":0.40000000000000002},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.433,"5min":0.42499999999999999,"15min":0.42299999999999999},"min":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.64000000000000001,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.41699999999999998}]},{"osd":1,"last update":"Tue Apr 2 15:01:10 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.438,"15min":0.441},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.44600000000000001,"15min":0.44400000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.47199999999999998}]},{"osd":2,"last update":"Tue Apr 2 15:01:10 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.47099999999999997,"15min":0.47299999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.56299999999999994,"5min":0.62,"15min":0.62},"last":0.375},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45100000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.56399999999999995,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.438}]},{"osd":4,"last update":"Tue Apr 2 15:01:23 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.57999999999999996,"15min":0.57899999999999996},"min":{"1min":0.39600000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.075,"5min":1.075,"15min":1.075},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59499999999999997,"15min":0.59699999999999998},"min":{"1min":0.42999999999999999,"5min":0.379,"15min":0.379},"max":{"1min":0.73199999999999998,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.58199999999999996}]},{"osd":5,"last update":"Tue Apr 2 15:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59999999999999998,"15min":0.60199999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.623,"5min":0.62,"15min":0.62},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.79700000000000004,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.59899999999999998}]},{"osd":6,"last update":"Tue Apr 2 15:01:10 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.624},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.51800000000000002}]},{"osd":7,"last update":"Tue Apr 2 15:01:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.54000000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986208,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:01:23 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.58699999999999997,"15min":0.59899999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.66000000000000003,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.60699999999999998,"15min":0.61699999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.68300000000000005,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.66700000000000004}]},{"osd":1,"last update":"Tue Apr 2 15:01:23 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.54700000000000004,"15min":0.55700000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.60899999999999999,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.498,"5min":0.55800000000000005,"15min":0.56799999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.63900000000000001,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.63}]},{"osd":2,"last update":"Tue Apr 2 15:01:23 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.56699999999999995,"15min":0.57899999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.74399999999999999,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.55900000000000005,"15min":0.56999999999999995},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.63100000000000001,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.623}]},{"osd":3,"last update":"Tue Apr 2 15:01:23 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.58499999999999996,"15min":0.60099999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.68600000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.55500000000000005,"15min":0.56100000000000005},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.64800000000000002,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.61099999999999999}]},{"osd":5,"last update":"Tue Apr 2 15:01:50 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.371,"15min":0.36499999999999999},"min":{"1min":0.29299999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.39300000000000002,"15min":0.377},"min":{"1min":0.30599999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.44600000000000001}]},{"osd":6,"last update":"Tue Apr 2 15:01:12 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.51500000000000001}]},{"osd":7,"last update":"Tue Apr 2 15:01:33 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.53400000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724572,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":240019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:01:52 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56799999999999995,"15min":0.57099999999999995},"min":{"1min":0.432,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.71899999999999997,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.54500000000000004,"15min":0.55000000000000004},"min":{"1min":0.38200000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.70999999999999996,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.45700000000000002}]},{"osd":1,"last update":"Tue Apr 2 15:01:52 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.55300000000000005,"15min":0.55400000000000005},"min":{"1min":0.443,"5min":0.313,"15min":0.313},"max":{"1min":0.68700000000000006,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.56599999999999995,"15min":0.57299999999999995},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.73099999999999998,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.51800000000000002}]},{"osd":2,"last update":"Tue Apr 2 15:01:52 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54500000000000004,"15min":0.55100000000000005},"min":{"1min":0.39500000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.69599999999999995,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.502,"5min":0.56599999999999995,"15min":0.57699999999999996},"min":{"1min":0.38200000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.68100000000000005,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.44600000000000001}]},{"osd":3,"last update":"Tue Apr 2 15:01:52 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.55000000000000004,"15min":0.55700000000000005},"min":{"1min":0.34999999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.628,"5min":0.748,"15min":0.748},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.57299999999999995,"15min":0.57999999999999996},"min":{"1min":0.40100000000000002,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.65800000000000003,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.40100000000000002}]},{"osd":4,"last update":"Tue Apr 2 15:01:52 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.33400000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.44,"5min":0.42199999999999999,"15min":0.41899999999999998},"min":{"1min":0.33500000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.47299999999999998}]},{"osd":6,"last update":"Tue Apr 2 15:01:10 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.60599999999999998}]},{"osd":7,"last update":"Tue Apr 2 15:01:32 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.63200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-02T15:02:00.023 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-02T15:02:00.023 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-02T15:02:00.023 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-02T15:02:00.023 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -- ceph health --format=json 2024-04-02T15:02:00.244 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:02:00.617 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:00 smithi042 ceph-mon[29473]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T15:02:00.617 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:00 smithi042 ceph-mon[24805]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T15:02:00.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:00 smithi169 ceph-mon[25801]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T15:02:01.719 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:01 smithi042 ceph-mon[24805]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:01.719 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:01 smithi042 ceph-mon[29473]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:01.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:01 smithi169 ceph-mon[25801]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:01.987 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:02:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:02:01] "GET /metrics HTTP/1.1" 200 191483 "" "Prometheus/2.33.4" 2024-04-02T15:02:02.430 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:02:02.430 INFO:teuthology.orchestra.run.smithi042.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-02T15:02:02.477 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:02:02 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:02:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:02:02.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:02 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3135785654' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-02T15:02:02.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:02 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3135785654' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-02T15:02:02.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:02 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3135785654' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-02T15:02:03.097 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-02T15:02:03.098 INFO:tasks.cephadm:Setup complete, yielding 2024-04-02T15:02:03.098 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T15:02:03.107 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi042.front.sepia.ceph.com 2024-04-02T15:02:03.107 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-04-02T15:02:03.765 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:03 smithi042 ceph-mon[24805]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:03.765 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:03 smithi042 ceph-mon[29473]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:03.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:03 smithi169 ceph-mon[25801]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:05.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:05 smithi042 ceph-mon[24805]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:05.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:05 smithi042 ceph-mon[29473]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:05.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:05 smithi169 ceph-mon[25801]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:06.547 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-04-02T15:02:06.547 INFO:teuthology.orchestra.run.smithi042.stdout: "id": "7382925c-61a3-4c54-a391-bb23f9cdfc14", 2024-04-02T15:02:06.547 INFO:teuthology.orchestra.run.smithi042.stdout: "name": "r", 2024-04-02T15:02:06.547 INFO:teuthology.orchestra.run.smithi042.stdout: "current_period": "c00a99e1-3b05-475d-bad3-f3b6c38a096c", 2024-04-02T15:02:06.547 INFO:teuthology.orchestra.run.smithi042.stdout: "epoch": 1 2024-04-02T15:02:06.547 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-04-02T15:02:06.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:06 smithi042 ceph-mon[24805]: osdmap e61: 8 total, 8 up, 8 in 2024-04-02T15:02:06.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:06 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3510935407' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-02T15:02:06.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:06 smithi042 ceph-mon[29473]: osdmap e61: 8 total, 8 up, 8 in 2024-04-02T15:02:06.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:06 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3510935407' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-02T15:02:06.794 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:06 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[47630]: level=info ts=2024-04-02T15:02:06.755Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002382838s 2024-04-02T15:02:06.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:06 smithi169 ceph-mon[25801]: osdmap e61: 8 total, 8 up, 8 in 2024-04-02T15:02:06.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:06 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3510935407' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-02T15:02:07.456 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-04-02T15:02:07.774 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:07 smithi042 ceph-mon[24805]: pgmap v43: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:07.774 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:07 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3510935407' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-02T15:02:07.774 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:07 smithi042 ceph-mon[24805]: osdmap e62: 8 total, 8 up, 8 in 2024-04-02T15:02:07.774 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:07 smithi042 ceph-mon[29473]: pgmap v43: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:07.774 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:07 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3510935407' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-02T15:02:07.774 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:07 smithi042 ceph-mon[29473]: osdmap e62: 8 total, 8 up, 8 in 2024-04-02T15:02:07.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:07 smithi169 ceph-mon[25801]: pgmap v43: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:07.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:07 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3510935407' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-02T15:02:07.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:07 smithi169 ceph-mon[25801]: osdmap e62: 8 total, 8 up, 8 in 2024-04-02T15:02:08.786 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:08 smithi042 ceph-mon[24805]: osdmap e63: 8 total, 8 up, 8 in 2024-04-02T15:02:08.787 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:08 smithi042 ceph-mon[29473]: osdmap e63: 8 total, 8 up, 8 in 2024-04-02T15:02:08.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:08 smithi169 ceph-mon[25801]: osdmap e63: 8 total, 8 up, 8 in 2024-04-02T15:02:09.096 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-04-02T15:02:09.097 INFO:teuthology.orchestra.run.smithi042.stdout: "id": "4004f290-15a9-4af0-85eb-6458fae687d9", 2024-04-02T15:02:09.097 INFO:teuthology.orchestra.run.smithi042.stdout: "name": "default", 2024-04-02T15:02:09.097 INFO:teuthology.orchestra.run.smithi042.stdout: "api_name": "default", 2024-04-02T15:02:09.097 INFO:teuthology.orchestra.run.smithi042.stdout: "is_master": "true", 2024-04-02T15:02:09.097 INFO:teuthology.orchestra.run.smithi042.stdout: "endpoints": [], 2024-04-02T15:02:09.097 INFO:teuthology.orchestra.run.smithi042.stdout: "hostnames": [], 2024-04-02T15:02:09.097 INFO:teuthology.orchestra.run.smithi042.stdout: "hostnames_s3website": [], 2024-04-02T15:02:09.097 INFO:teuthology.orchestra.run.smithi042.stdout: "master_zone": "", 2024-04-02T15:02:09.097 INFO:teuthology.orchestra.run.smithi042.stdout: "zones": [], 2024-04-02T15:02:09.097 INFO:teuthology.orchestra.run.smithi042.stdout: "placement_targets": [], 2024-04-02T15:02:09.097 INFO:teuthology.orchestra.run.smithi042.stdout: "default_placement": "", 2024-04-02T15:02:09.097 INFO:teuthology.orchestra.run.smithi042.stdout: "realm_id": "7382925c-61a3-4c54-a391-bb23f9cdfc14", 2024-04-02T15:02:09.097 INFO:teuthology.orchestra.run.smithi042.stdout: "sync_policy": { 2024-04-02T15:02:09.098 INFO:teuthology.orchestra.run.smithi042.stdout: "groups": [] 2024-04-02T15:02:09.098 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-04-02T15:02:09.098 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-04-02T15:02:09.687 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-04-02T15:02:09.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:09 smithi042 ceph-mon[24805]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:09.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:09 smithi042 ceph-mon[29473]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:09.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:09 smithi169 ceph-mon[25801]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:11.268 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-04-02T15:02:11.268 INFO:teuthology.orchestra.run.smithi042.stdout: "id": "97721a22-a7b1-46be-952c-63424359a4a7", 2024-04-02T15:02:11.268 INFO:teuthology.orchestra.run.smithi042.stdout: "name": "z", 2024-04-02T15:02:11.268 INFO:teuthology.orchestra.run.smithi042.stdout: "domain_root": "z.rgw.meta:root", 2024-04-02T15:02:11.268 INFO:teuthology.orchestra.run.smithi042.stdout: "control_pool": "z.rgw.control", 2024-04-02T15:02:11.268 INFO:teuthology.orchestra.run.smithi042.stdout: "gc_pool": "z.rgw.log:gc", 2024-04-02T15:02:11.268 INFO:teuthology.orchestra.run.smithi042.stdout: "lc_pool": "z.rgw.log:lc", 2024-04-02T15:02:11.269 INFO:teuthology.orchestra.run.smithi042.stdout: "log_pool": "z.rgw.log", 2024-04-02T15:02:11.269 INFO:teuthology.orchestra.run.smithi042.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-04-02T15:02:11.269 INFO:teuthology.orchestra.run.smithi042.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-04-02T15:02:11.269 INFO:teuthology.orchestra.run.smithi042.stdout: "roles_pool": "z.rgw.meta:roles", 2024-04-02T15:02:11.269 INFO:teuthology.orchestra.run.smithi042.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-04-02T15:02:11.269 INFO:teuthology.orchestra.run.smithi042.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-04-02T15:02:11.269 INFO:teuthology.orchestra.run.smithi042.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-04-02T15:02:11.269 INFO:teuthology.orchestra.run.smithi042.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-04-02T15:02:11.269 INFO:teuthology.orchestra.run.smithi042.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-04-02T15:02:11.269 INFO:teuthology.orchestra.run.smithi042.stdout: "otp_pool": "z.rgw.otp", 2024-04-02T15:02:11.269 INFO:teuthology.orchestra.run.smithi042.stdout: "system_key": { 2024-04-02T15:02:11.269 INFO:teuthology.orchestra.run.smithi042.stdout: "access_key": "", 2024-04-02T15:02:11.269 INFO:teuthology.orchestra.run.smithi042.stdout: "secret_key": "" 2024-04-02T15:02:11.269 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:02:11.270 INFO:teuthology.orchestra.run.smithi042.stdout: "placement_pools": [ 2024-04-02T15:02:11.270 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-04-02T15:02:11.270 INFO:teuthology.orchestra.run.smithi042.stdout: "key": "default-placement", 2024-04-02T15:02:11.270 INFO:teuthology.orchestra.run.smithi042.stdout: "val": { 2024-04-02T15:02:11.270 INFO:teuthology.orchestra.run.smithi042.stdout: "index_pool": "z.rgw.buckets.index", 2024-04-02T15:02:11.270 INFO:teuthology.orchestra.run.smithi042.stdout: "storage_classes": { 2024-04-02T15:02:11.270 INFO:teuthology.orchestra.run.smithi042.stdout: "STANDARD": { 2024-04-02T15:02:11.270 INFO:teuthology.orchestra.run.smithi042.stdout: "data_pool": "z.rgw.buckets.data" 2024-04-02T15:02:11.270 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-04-02T15:02:11.270 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:02:11.270 INFO:teuthology.orchestra.run.smithi042.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-04-02T15:02:11.270 INFO:teuthology.orchestra.run.smithi042.stdout: "index_type": 0 2024-04-02T15:02:11.270 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-04-02T15:02:11.270 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-04-02T15:02:11.270 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-04-02T15:02:11.271 INFO:teuthology.orchestra.run.smithi042.stdout: "realm_id": "7382925c-61a3-4c54-a391-bb23f9cdfc14", 2024-04-02T15:02:11.271 INFO:teuthology.orchestra.run.smithi042.stdout: "notif_pool": "z.rgw.log:notif" 2024-04-02T15:02:11.271 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-04-02T15:02:11.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:11 smithi042 ceph-mon[24805]: pgmap v47: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-04-02T15:02:11.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:11 smithi042 ceph-mon[29473]: pgmap v47: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-04-02T15:02:11.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:11 smithi169 ceph-mon[25801]: pgmap v47: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-04-02T15:02:11.847 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-04-02T15:02:12.156 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:02:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:02:11] "GET /metrics HTTP/1.1" 200 191495 "" "Prometheus/2.33.4" 2024-04-02T15:02:12.477 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:02:12 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:02:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:02:13.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:13 smithi169 ceph-mon[25801]: pgmap v48: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 591 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-04-02T15:02:14.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:13 smithi042 ceph-mon[29473]: pgmap v48: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 591 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-04-02T15:02:14.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:13 smithi042 ceph-mon[24805]: pgmap v48: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 591 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-04-02T15:02:15.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:14 smithi042 ceph-mon[29473]: osdmap e64: 8 total, 8 up, 8 in 2024-04-02T15:02:15.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:14 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/2066489416' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-02T15:02:15.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:14 smithi042 ceph-mon[29473]: pgmap v50: 65 pgs: 32 unknown, 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 517 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-04-02T15:02:15.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:14 smithi042 ceph-mon[24805]: osdmap e64: 8 total, 8 up, 8 in 2024-04-02T15:02:15.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:14 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/2066489416' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-02T15:02:15.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:14 smithi042 ceph-mon[24805]: pgmap v50: 65 pgs: 32 unknown, 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 517 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-04-02T15:02:15.049 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:14 smithi169 ceph-mon[25801]: osdmap e64: 8 total, 8 up, 8 in 2024-04-02T15:02:15.049 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:14 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/2066489416' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-02T15:02:15.049 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:14 smithi169 ceph-mon[25801]: pgmap v50: 65 pgs: 32 unknown, 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 517 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-04-02T15:02:16.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:15 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/2066489416' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-02T15:02:16.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:15 smithi042 ceph-mon[24805]: osdmap e65: 8 total, 8 up, 8 in 2024-04-02T15:02:16.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:15 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/2066489416' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-02T15:02:16.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:15 smithi042 ceph-mon[29473]: osdmap e65: 8 total, 8 up, 8 in 2024-04-02T15:02:16.089 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:15 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/2066489416' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-02T15:02:16.089 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:15 smithi169 ceph-mon[25801]: osdmap e65: 8 total, 8 up, 8 in 2024-04-02T15:02:16.727 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 podman[41905]: 2024-04-02 15:02:16.580592366 +0000 UTC m=+0.017158288 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-02T15:02:17.005 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-mon[25801]: osdmap e66: 8 total, 8 up, 8 in 2024-04-02T15:02:17.005 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/2066489416' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-02T15:02:17.005 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-mon[25801]: pgmap v53: 97 pgs: 32 unknown, 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 853 B/s wr, 5 op/s 2024-04-02T15:02:17.006 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 podman[41905]: 2024-04-02 15:02:16.888721614 +0000 UTC m=+0.325287523 container init acd65a3d3b6af3a42f8f7a492670762afce133d0562c32a0270c7d9474e1a902 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a, distribution-scope=public, com.redhat.component=ubi8-container, architecture=x86_64, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.openshift.expose-services=, name=ubi8, release=236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, maintainer=Paul Cuzner , vendor=Red Hat, Inc., version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.24.2) 2024-04-02T15:02:17.006 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 podman[41905]: 2024-04-02 15:02:16.891577242 +0000 UTC m=+0.328143148 container start acd65a3d3b6af3a42f8f7a492670762afce133d0562c32a0270c7d9474e1a902 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a, name=ubi8, description=Ceph Grafana Container, io.openshift.tags=base rhel8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, release=236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.24.2, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-03-28T10:36:18.413762, com.redhat.component=ubi8-container, version=8.5, maintainer=Paul Cuzner , vendor=Red Hat, Inc., architecture=x86_64) 2024-04-02T15:02:17.006 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-04-02T15:02:17.006 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-04-02T15:02:17.006 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-04-02T15:02:17.006 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-04-02T15:02:17.007 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-02T15:02:17.007 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-02T15:02:17.007 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-02T15:02:17.007 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-02T15:02:17.007 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-04-02T15:02:17.007 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-04-02T15:02:17.007 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-04-02T15:02:17.007 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-04-02T15:02:17.007 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-04-02T15:02:17.007 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="App mode production" logger=settings 2024-04-02T15:02:17.007 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-04-02T15:02:17.007 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-02T15:02:17.007 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-04-02T15:02:17.007 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-04-02T15:02:17.007 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-04-02T15:02:17.008 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-04-02T15:02:17.008 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-04-02T15:02:17.008 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-04-02T15:02:17.008 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-04-02T15:02:17.008 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-04-02T15:02:17.008 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-04-02T15:02:17.008 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-04-02T15:02:17.008 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-04-02T15:02:17.008 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-04-02T15:02:17.008 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-04-02T15:02:17.008 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-04-02T15:02:17.008 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-04-02T15:02:17.008 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-04-02T15:02:17.008 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-04-02T15:02:17.008 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-04-02T15:02:17.009 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-04-02T15:02:17.009 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-04-02T15:02:17.009 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-04-02T15:02:17.009 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-04-02T15:02:17.009 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-04-02T15:02:17.009 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-04-02T15:02:17.009 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-04-02T15:02:17.009 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-04-02T15:02:17.009 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-04-02T15:02:17.009 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-04-02T15:02:17.009 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-04-02T15:02:17.010 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-04-02T15:02:17.010 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-02T15:02:17.010 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-04-02T15:02:17.010 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-04-02T15:02:17.010 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-04-02T15:02:17.010 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-04-02T15:02:17.010 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-04-02T15:02:17.010 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-04-02T15:02:17.010 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-04-02T15:02:17.011 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-04-02T15:02:17.011 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-04-02T15:02:17.011 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-04-02T15:02:17.011 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-04-02T15:02:17.011 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-04-02T15:02:17.011 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-04-02T15:02:17.011 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-04-02T15:02:17.011 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-04-02T15:02:17.011 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 bash[41905]: acd65a3d3b6af3a42f8f7a492670762afce133d0562c32a0270c7d9474e1a902 2024-04-02T15:02:17.011 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-04-02T15:02:17.011 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 systemd[1]: Started Ceph grafana.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T15:02:17.011 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-04-02T15:02:17.011 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-04-02T15:02:17.011 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-04-02T15:02:17.011 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-04-02T15:02:17.012 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-04-02T15:02:17.012 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-04-02T15:02:17.012 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-04-02T15:02:17.012 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-04-02T15:02:17.012 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-02T15:02:17.012 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-04-02T15:02:17.012 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-04-02T15:02:17.012 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-04-02T15:02:17.012 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-04-02T15:02:17.012 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-04-02T15:02:17.012 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-04-02T15:02:17.012 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-04-02T15:02:17.012 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-04-02T15:02:17.012 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-04-02T15:02:17.012 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-04-02T15:02:17.013 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-04-02T15:02:17.013 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-04-02T15:02:17.013 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-04-02T15:02:17.013 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-04-02T15:02:17.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:16 smithi042 ceph-mon[24805]: osdmap e66: 8 total, 8 up, 8 in 2024-04-02T15:02:17.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:16 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/2066489416' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-02T15:02:17.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:16 smithi042 ceph-mon[24805]: pgmap v53: 97 pgs: 32 unknown, 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 853 B/s wr, 5 op/s 2024-04-02T15:02:17.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:16 smithi042 ceph-mon[29473]: osdmap e66: 8 total, 8 up, 8 in 2024-04-02T15:02:17.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:16 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/2066489416' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-02T15:02:17.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:16 smithi042 ceph-mon[29473]: pgmap v53: 97 pgs: 32 unknown, 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 853 B/s wr, 5 op/s 2024-04-02T15:02:17.258 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-04-02T15:02:17.258 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-04-02T15:02:17.259 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-04-02T15:02:17.259 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-04-02T15:02:17.259 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-04-02T15:02:17.259 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-04-02T15:02:17.259 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-04-02T15:02:17.259 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-04-02T15:02:17.259 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-04-02T15:02:17.259 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-04-02T15:02:17.259 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-02T15:02:17.259 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-04-02T15:02:17.259 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-02T15:02:17.259 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-04-02T15:02:17.259 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-02T15:02:17.259 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-02T15:02:17.260 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-04-02T15:02:17.260 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-04-02T15:02:17.260 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-04-02T15:02:17.261 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-04-02T15:02:17.261 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-04-02T15:02:17.261 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-04-02T15:02:17.261 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-04-02T15:02:17.261 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-04-02T15:02:17.261 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-04-02T15:02:17.261 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-04-02T15:02:17.261 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-04-02T15:02:17.261 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-04-02T15:02:17.261 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-04-02T15:02:17.261 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-04-02T15:02:17.262 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-04-02T15:02:17.262 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-04-02T15:02:17.262 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-04-02T15:02:17.262 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-04-02T15:02:17.262 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-04-02T15:02:17.262 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-04-02T15:02:17.262 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-04-02T15:02:17.262 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-04-02T15:02:17.262 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-04-02T15:02:17.262 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-04-02T15:02:17.262 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-04-02T15:02:17.262 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-04-02T15:02:17.262 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-04-02T15:02:17.262 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-04-02T15:02:17.263 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-04-02T15:02:17.263 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-04-02T15:02:17.263 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-04-02T15:02:17.263 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-04-02T15:02:17.263 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-04-02T15:02:17.263 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-04-02T15:02:17.263 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-04-02T15:02:17.263 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-04-02T15:02:17.263 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-04-02T15:02:17.263 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-04-02T15:02:17.263 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-04-02T15:02:17.263 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-04-02T15:02:17.263 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-04-02T15:02:17.263 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-04-02T15:02:17.263 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-04-02T15:02:17.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-04-02T15:02:17.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-04-02T15:02:17.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-04-02T15:02:17.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-04-02T15:02:17.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-04-02T15:02:17.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-04-02T15:02:17.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-04-02T15:02:17.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-02T15:02:17.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-02T15:02:17.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-04-02T15:02:17.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-04-02T15:02:17.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-04-02T15:02:17.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-04-02T15:02:17.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-02T15:02:17.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-04-02T15:02:17.265 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-02T15:02:17.265 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-04-02T15:02:17.265 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-04-02T15:02:17.265 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-02T15:02:17.265 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-04-02T15:02:17.265 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-04-02T15:02:17.265 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-04-02T15:02:17.265 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-04-02T15:02:17.265 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-04-02T15:02:17.265 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-04-02T15:02:17.265 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-04-02T15:02:17.265 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-04-02T15:02:17.265 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-04-02T15:02:17.265 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-04-02T15:02:17.265 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-04-02T15:02:17.266 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-04-02T15:02:17.266 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-04-02T15:02:17.266 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-04-02T15:02:17.266 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-04-02T15:02:17.266 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-04-02T15:02:17.266 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-04-02T15:02:17.266 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-04-02T15:02:17.266 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-04-02T15:02:17.266 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-04-02T15:02:17.266 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-04-02T15:02:17.266 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-02T15:02:17.267 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-02T15:02:17.267 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-02T15:02:17.267 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-04-02T15:02:17.268 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-02T15:02:17.268 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-04-02T15:02:17.268 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-04-02T15:02:17.268 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-04-02T15:02:17.268 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-04-02T15:02:17.268 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-04-02T15:02:17.268 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-04-02T15:02:17.268 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-04-02T15:02:17.268 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-04-02T15:02:17.268 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-04-02T15:02:17.268 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-04-02T15:02:17.268 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-04-02T15:02:17.268 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-02T15:02:17.268 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-04-02T15:02:17.269 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-04-02T15:02:17.269 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-02T15:02:17.269 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-04-02T15:02:17.269 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-04-02T15:02:17.269 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-04-02T15:02:17.269 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-04-02T15:02:17.269 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-04-02T15:02:17.269 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-04-02T15:02:17.269 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-04-02T15:02:17.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-04-02T15:02:17.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-04-02T15:02:17.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-04-02T15:02:17.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-04-02T15:02:17.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-04-02T15:02:17.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-04-02T15:02:17.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-04-02T15:02:17.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-04-02T15:02:17.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-04-02T15:02:17.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-04-02T15:02:17.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-04-02T15:02:17.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-04-02T15:02:17.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-04-02T15:02:17.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-04-02T15:02:17.271 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-02T15:02:17.271 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-02T15:02:17.271 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-02T15:02:17.271 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-04-02T15:02:17.271 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-02T15:02:17.271 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-04-02T15:02:17.271 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-04-02T15:02:17.271 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-04-02T15:02:17.271 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-04-02T15:02:17.271 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-04-02T15:02:17.271 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-04-02T15:02:17.271 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-04-02T15:02:17.271 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-04-02T15:02:17.271 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-04-02T15:02:17.272 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-04-02T15:02:17.272 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-04-02T15:02:17.272 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-04-02T15:02:17.272 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-04-02T15:02:17.272 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-02T15:02:17.272 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-02T15:02:17.272 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-02T15:02:17.272 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-02T15:02:17.272 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-04-02T15:02:17.272 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-04-02T15:02:17.272 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-04-02T15:02:17.272 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-04-02T15:02:17.272 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-02T15:02:17.272 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-02T15:02:17.273 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-02T15:02:17.274 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-04-02T15:02:17.274 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-04-02T15:02:17.274 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-04-02T15:02:17.274 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-04-02T15:02:17.274 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-04-02T15:02:17.274 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-04-02T15:02:17.274 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-04-02T15:02:17.274 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-04-02T15:02:17.274 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-04-02T15:02:17.274 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-04-02T15:02:17.274 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-04-02T15:02:17.274 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-02T15:02:17.274 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-04-02T15:02:17.274 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-04-02T15:02:17.275 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-04-02T15:02:17.275 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-04-02T15:02:17.275 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-04-02T15:02:17.275 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-02T15:02:17.533 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-04-02T15:02:17.533 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-04-02T15:02:17.533 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-04-02T15:02:17.533 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-04-02T15:02:17.533 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-04-02T15:02:17.533 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-04-02T15:02:17.534 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-04-02T15:02:17.534 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-04-02T15:02:17.534 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-04-02T15:02:17.534 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-04-02T15:02:17.534 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-04-02T15:02:17.534 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-04-02T15:02:17.534 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-04-02T15:02:17.534 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-04-02T15:02:17.534 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-04-02T15:02:17.534 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-04-02T15:02:17.534 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-04-02T15:02:17.534 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-04-02T15:02:17.534 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-04-02T15:02:17.534 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-04-02T15:02:17.534 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-04-02T15:02:17.535 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-04-02T15:02:17.535 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-04-02T15:02:17.535 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-04-02T15:02:17.535 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-04-02T15:02:17.535 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-04-02T15:02:17.535 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-04-02T15:02:17.535 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-04-02T15:02:17.535 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-04-02T15:02:17.535 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-04-02T15:02:17.535 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-02T15:02:17.535 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-02T15:02:17.535 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-02T15:02:17.535 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-04-02T15:02:17.535 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-04-02T15:02:17.535 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-02T15:02:17.536 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-02T15:02:17.536 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-04-02T15:02:17.536 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-02T15:02:17.536 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-04-02T15:02:17.536 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-04-02T15:02:17.536 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-04-02T15:02:17.536 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-02T15:02:17.536 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-04-02T15:02:17.536 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-04-02T15:02:17.536 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-04-02T15:02:17.536 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-04-02T15:02:17.536 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-02T15:02:17.536 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-04-02T15:02:17.536 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-04-02T15:02:17.536 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-04-02T15:02:17.537 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-04-02T15:02:17.537 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-04-02T15:02:17.537 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-02T15:02:17.537 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-04-02T15:02:17.537 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-04-02T15:02:17.537 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-02T15:02:17.537 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-04-02T15:02:17.538 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-02T15:02:17.538 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-02T15:02:17.538 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-02T15:02:17.538 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-04-02T15:02:17.538 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-04-02T15:02:17.538 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-04-02T15:02:17.538 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-04-02T15:02:17.538 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-04-02T15:02:17.538 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-02T15:02:17.538 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-04-02T15:02:17.538 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-04-02T15:02:17.538 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-04-02T15:02:17.538 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-04-02T15:02:17.538 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-04-02T15:02:17.539 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-04-02T15:02:17.540 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-04-02T15:02:17.540 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-04-02T15:02:17.540 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-04-02T15:02:17.540 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-04-02T15:02:17.540 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-04-02T15:02:17.540 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-04-02T15:02:17.540 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-04-02T15:02:17.540 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-04-02T15:02:17.540 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-04-02T15:02:17.540 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-04-02T15:02:17.540 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-04-02T15:02:17.540 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-04-02T15:02:17.540 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-04-02T15:02:17.540 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-04-02T15:02:17.540 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-04-02T15:02:17.541 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-04-02T15:02:17.541 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-04-02T15:02:17.541 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-04-02T15:02:17.541 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-04-02T15:02:17.541 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-04-02T15:02:17.541 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-04-02T15:02:17.541 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-04-02T15:02:17.541 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=577.372736ms 2024-04-02T15:02:17.541 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-04-02T15:02:17.541 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-04-02T15:02:17.541 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-04-02T15:02:17.541 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-04-02T15:02:17.541 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-04-02T15:02:17.787 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/2066489416' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-02T15:02:17.787 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-mon[25801]: osdmap e67: 8 total, 8 up, 8 in 2024-04-02T15:02:17.787 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:17.787 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:02:17.787 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:17.787 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:02:17.787 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-04-02T15:02:17.788 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-02T15:02:17.788 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-02T15:02:17.788 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-04-02T15:02:17.788 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-04-02T15:02:17.788 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:02:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:02:17+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-04-02T15:02:18.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:17 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/2066489416' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-02T15:02:18.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:17 smithi042 ceph-mon[24805]: osdmap e67: 8 total, 8 up, 8 in 2024-04-02T15:02:18.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:17 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:18.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:17 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:02:18.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:17 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:18.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:17 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:02:18.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:17 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/2066489416' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-02T15:02:18.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:17 smithi042 ceph-mon[29473]: osdmap e67: 8 total, 8 up, 8 in 2024-04-02T15:02:18.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:17 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:18.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:17 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:02:18.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:17 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:18.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:17 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:02:18.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:18 smithi169 ceph-mon[25801]: osdmap e68: 8 total, 8 up, 8 in 2024-04-02T15:02:18.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:18 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/507882436' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T15:02:18.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:18 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T15:02:18.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:18 smithi169 ceph-mon[25801]: pgmap v56: 129 pgs: 64 unknown, 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-04-02T15:02:18.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:18 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:19.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:18 smithi042 ceph-mon[24805]: osdmap e68: 8 total, 8 up, 8 in 2024-04-02T15:02:19.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:18 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/507882436' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T15:02:19.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:18 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T15:02:19.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:18 smithi042 ceph-mon[24805]: pgmap v56: 129 pgs: 64 unknown, 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-04-02T15:02:19.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:18 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:19.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:18 smithi042 ceph-mon[29473]: osdmap e68: 8 total, 8 up, 8 in 2024-04-02T15:02:19.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:18 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/507882436' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T15:02:19.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:18 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T15:02:19.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:18 smithi042 ceph-mon[29473]: pgmap v56: 129 pgs: 64 unknown, 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-04-02T15:02:19.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:18 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:20.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:19 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-02T15:02:20.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:19 smithi042 ceph-mon[29473]: osdmap e69: 8 total, 8 up, 8 in 2024-04-02T15:02:20.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:19 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/507882436' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T15:02:20.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:19 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T15:02:20.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:19 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:20.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:19 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-02T15:02:20.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:19 smithi042 ceph-mon[24805]: osdmap e69: 8 total, 8 up, 8 in 2024-04-02T15:02:20.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:19 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/507882436' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T15:02:20.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:19 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T15:02:20.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:19 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:20.092 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:19 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-02T15:02:20.092 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:19 smithi169 ceph-mon[25801]: osdmap e69: 8 total, 8 up, 8 in 2024-04-02T15:02:20.092 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:19 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/507882436' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T15:02:20.092 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:19 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T15:02:20.093 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:19 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:20.829 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-04-02T15:02:20.829 INFO:teuthology.orchestra.run.smithi042.stdout: "id": "4db872f1-196a-4850-8ada-acaf2b09b27b", 2024-04-02T15:02:20.829 INFO:teuthology.orchestra.run.smithi042.stdout: "epoch": 1, 2024-04-02T15:02:20.829 INFO:teuthology.orchestra.run.smithi042.stdout: "predecessor_uuid": "c00a99e1-3b05-475d-bad3-f3b6c38a096c", 2024-04-02T15:02:20.829 INFO:teuthology.orchestra.run.smithi042.stdout: "sync_status": [], 2024-04-02T15:02:20.829 INFO:teuthology.orchestra.run.smithi042.stdout: "period_map": { 2024-04-02T15:02:20.830 INFO:teuthology.orchestra.run.smithi042.stdout: "id": "4db872f1-196a-4850-8ada-acaf2b09b27b", 2024-04-02T15:02:20.830 INFO:teuthology.orchestra.run.smithi042.stdout: "zonegroups": [ 2024-04-02T15:02:20.830 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-04-02T15:02:20.830 INFO:teuthology.orchestra.run.smithi042.stdout: "id": "4004f290-15a9-4af0-85eb-6458fae687d9", 2024-04-02T15:02:20.830 INFO:teuthology.orchestra.run.smithi042.stdout: "name": "default", 2024-04-02T15:02:20.830 INFO:teuthology.orchestra.run.smithi042.stdout: "api_name": "default", 2024-04-02T15:02:20.830 INFO:teuthology.orchestra.run.smithi042.stdout: "is_master": "true", 2024-04-02T15:02:20.830 INFO:teuthology.orchestra.run.smithi042.stdout: "endpoints": [], 2024-04-02T15:02:20.830 INFO:teuthology.orchestra.run.smithi042.stdout: "hostnames": [], 2024-04-02T15:02:20.830 INFO:teuthology.orchestra.run.smithi042.stdout: "hostnames_s3website": [], 2024-04-02T15:02:20.830 INFO:teuthology.orchestra.run.smithi042.stdout: "master_zone": "97721a22-a7b1-46be-952c-63424359a4a7", 2024-04-02T15:02:20.830 INFO:teuthology.orchestra.run.smithi042.stdout: "zones": [ 2024-04-02T15:02:20.830 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-04-02T15:02:20.830 INFO:teuthology.orchestra.run.smithi042.stdout: "id": "97721a22-a7b1-46be-952c-63424359a4a7", 2024-04-02T15:02:20.831 INFO:teuthology.orchestra.run.smithi042.stdout: "name": "z", 2024-04-02T15:02:20.831 INFO:teuthology.orchestra.run.smithi042.stdout: "endpoints": [], 2024-04-02T15:02:20.831 INFO:teuthology.orchestra.run.smithi042.stdout: "log_meta": "false", 2024-04-02T15:02:20.831 INFO:teuthology.orchestra.run.smithi042.stdout: "log_data": "false", 2024-04-02T15:02:20.831 INFO:teuthology.orchestra.run.smithi042.stdout: "bucket_index_max_shards": 11, 2024-04-02T15:02:20.831 INFO:teuthology.orchestra.run.smithi042.stdout: "read_only": "false", 2024-04-02T15:02:20.831 INFO:teuthology.orchestra.run.smithi042.stdout: "tier_type": "", 2024-04-02T15:02:20.831 INFO:teuthology.orchestra.run.smithi042.stdout: "sync_from_all": "true", 2024-04-02T15:02:20.831 INFO:teuthology.orchestra.run.smithi042.stdout: "sync_from": [], 2024-04-02T15:02:20.831 INFO:teuthology.orchestra.run.smithi042.stdout: "redirect_zone": "" 2024-04-02T15:02:20.831 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-04-02T15:02:20.831 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-04-02T15:02:20.831 INFO:teuthology.orchestra.run.smithi042.stdout: "placement_targets": [ 2024-04-02T15:02:20.831 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-04-02T15:02:20.831 INFO:teuthology.orchestra.run.smithi042.stdout: "name": "default-placement", 2024-04-02T15:02:20.832 INFO:teuthology.orchestra.run.smithi042.stdout: "tags": [], 2024-04-02T15:02:20.832 INFO:teuthology.orchestra.run.smithi042.stdout: "storage_classes": [ 2024-04-02T15:02:20.832 INFO:teuthology.orchestra.run.smithi042.stdout: "STANDARD" 2024-04-02T15:02:20.832 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-04-02T15:02:20.832 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-04-02T15:02:20.832 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-04-02T15:02:20.832 INFO:teuthology.orchestra.run.smithi042.stdout: "default_placement": "default-placement", 2024-04-02T15:02:20.832 INFO:teuthology.orchestra.run.smithi042.stdout: "realm_id": "7382925c-61a3-4c54-a391-bb23f9cdfc14", 2024-04-02T15:02:20.832 INFO:teuthology.orchestra.run.smithi042.stdout: "sync_policy": { 2024-04-02T15:02:20.832 INFO:teuthology.orchestra.run.smithi042.stdout: "groups": [] 2024-04-02T15:02:20.832 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-04-02T15:02:20.832 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-04-02T15:02:20.832 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-04-02T15:02:20.832 INFO:teuthology.orchestra.run.smithi042.stdout: "short_zone_ids": [ 2024-04-02T15:02:20.833 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-04-02T15:02:20.833 INFO:teuthology.orchestra.run.smithi042.stdout: "key": "97721a22-a7b1-46be-952c-63424359a4a7", 2024-04-02T15:02:20.833 INFO:teuthology.orchestra.run.smithi042.stdout: "val": 3375835446 2024-04-02T15:02:20.833 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-04-02T15:02:20.833 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-04-02T15:02:20.833 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:02:20.833 INFO:teuthology.orchestra.run.smithi042.stdout: "master_zonegroup": "4004f290-15a9-4af0-85eb-6458fae687d9", 2024-04-02T15:02:20.833 INFO:teuthology.orchestra.run.smithi042.stdout: "master_zone": "97721a22-a7b1-46be-952c-63424359a4a7", 2024-04-02T15:02:20.833 INFO:teuthology.orchestra.run.smithi042.stdout: "period_config": { 2024-04-02T15:02:20.833 INFO:teuthology.orchestra.run.smithi042.stdout: "bucket_quota": { 2024-04-02T15:02:20.833 INFO:teuthology.orchestra.run.smithi042.stdout: "enabled": false, 2024-04-02T15:02:20.833 INFO:teuthology.orchestra.run.smithi042.stdout: "check_on_raw": false, 2024-04-02T15:02:20.833 INFO:teuthology.orchestra.run.smithi042.stdout: "max_size": -1, 2024-04-02T15:02:20.833 INFO:teuthology.orchestra.run.smithi042.stdout: "max_size_kb": 0, 2024-04-02T15:02:20.833 INFO:teuthology.orchestra.run.smithi042.stdout: "max_objects": -1 2024-04-02T15:02:20.834 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:02:20.834 INFO:teuthology.orchestra.run.smithi042.stdout: "user_quota": { 2024-04-02T15:02:20.834 INFO:teuthology.orchestra.run.smithi042.stdout: "enabled": false, 2024-04-02T15:02:20.834 INFO:teuthology.orchestra.run.smithi042.stdout: "check_on_raw": false, 2024-04-02T15:02:20.834 INFO:teuthology.orchestra.run.smithi042.stdout: "max_size": -1, 2024-04-02T15:02:20.834 INFO:teuthology.orchestra.run.smithi042.stdout: "max_size_kb": 0, 2024-04-02T15:02:20.834 INFO:teuthology.orchestra.run.smithi042.stdout: "max_objects": -1 2024-04-02T15:02:20.834 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:02:20.834 INFO:teuthology.orchestra.run.smithi042.stdout: "user_ratelimit": { 2024-04-02T15:02:20.834 INFO:teuthology.orchestra.run.smithi042.stdout: "max_read_ops": 0, 2024-04-02T15:02:20.834 INFO:teuthology.orchestra.run.smithi042.stdout: "max_write_ops": 0, 2024-04-02T15:02:20.834 INFO:teuthology.orchestra.run.smithi042.stdout: "max_read_bytes": 0, 2024-04-02T15:02:20.834 INFO:teuthology.orchestra.run.smithi042.stdout: "max_write_bytes": 0, 2024-04-02T15:02:20.834 INFO:teuthology.orchestra.run.smithi042.stdout: "enabled": false 2024-04-02T15:02:20.834 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:02:20.835 INFO:teuthology.orchestra.run.smithi042.stdout: "bucket_ratelimit": { 2024-04-02T15:02:20.835 INFO:teuthology.orchestra.run.smithi042.stdout: "max_read_ops": 0, 2024-04-02T15:02:20.835 INFO:teuthology.orchestra.run.smithi042.stdout: "max_write_ops": 0, 2024-04-02T15:02:20.835 INFO:teuthology.orchestra.run.smithi042.stdout: "max_read_bytes": 0, 2024-04-02T15:02:20.835 INFO:teuthology.orchestra.run.smithi042.stdout: "max_write_bytes": 0, 2024-04-02T15:02:20.835 INFO:teuthology.orchestra.run.smithi042.stdout: "enabled": false 2024-04-02T15:02:20.835 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:02:20.835 INFO:teuthology.orchestra.run.smithi042.stdout: "anonymous_ratelimit": { 2024-04-02T15:02:20.835 INFO:teuthology.orchestra.run.smithi042.stdout: "max_read_ops": 0, 2024-04-02T15:02:20.835 INFO:teuthology.orchestra.run.smithi042.stdout: "max_write_ops": 0, 2024-04-02T15:02:20.835 INFO:teuthology.orchestra.run.smithi042.stdout: "max_read_bytes": 0, 2024-04-02T15:02:20.835 INFO:teuthology.orchestra.run.smithi042.stdout: "max_write_bytes": 0, 2024-04-02T15:02:20.835 INFO:teuthology.orchestra.run.smithi042.stdout: "enabled": false 2024-04-02T15:02:20.835 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-04-02T15:02:20.836 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:02:20.836 INFO:teuthology.orchestra.run.smithi042.stdout: "realm_id": "7382925c-61a3-4c54-a391-bb23f9cdfc14", 2024-04-02T15:02:20.836 INFO:teuthology.orchestra.run.smithi042.stdout: "realm_name": "r", 2024-04-02T15:02:20.836 INFO:teuthology.orchestra.run.smithi042.stdout: "realm_epoch": 2 2024-04-02T15:02:20.836 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-04-02T15:02:20.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:20 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-02T15:02:20.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:20 smithi169 ceph-mon[25801]: osdmap e70: 8 total, 8 up, 8 in 2024-04-02T15:02:20.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:20 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/507882436' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T15:02:20.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:20 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T15:02:20.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:20 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:20.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:20 smithi169 ceph-mon[25801]: pgmap v59: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-02T15:02:20.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:20 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:21.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:20 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-02T15:02:21.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:20 smithi042 ceph-mon[24805]: osdmap e70: 8 total, 8 up, 8 in 2024-04-02T15:02:21.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:20 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/507882436' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T15:02:21.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:20 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T15:02:21.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:20 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:21.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:20 smithi042 ceph-mon[24805]: pgmap v59: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-02T15:02:21.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:20 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:21.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:20 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-02T15:02:21.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:20 smithi042 ceph-mon[29473]: osdmap e70: 8 total, 8 up, 8 in 2024-04-02T15:02:21.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:20 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/507882436' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T15:02:21.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:20 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T15:02:21.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:20 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:21.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:20 smithi042 ceph-mon[29473]: pgmap v59: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-02T15:02:21.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:20 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:21.578 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-04-02T15:02:21.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:21 smithi042 ceph-mon[29473]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-02T15:02:21.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:21 smithi042 ceph-mon[29473]: Reconfiguring daemon alertmanager.a on smithi042 2024-04-02T15:02:21.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:21 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-02T15:02:21.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:21 smithi042 ceph-mon[29473]: osdmap e71: 8 total, 8 up, 8 in 2024-04-02T15:02:21.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:21 smithi042 ceph-mon[24805]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-02T15:02:21.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:21 smithi042 ceph-mon[24805]: Reconfiguring daemon alertmanager.a on smithi042 2024-04-02T15:02:21.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:21 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-02T15:02:21.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:21 smithi042 ceph-mon[24805]: osdmap e71: 8 total, 8 up, 8 in 2024-04-02T15:02:22.152 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:02:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:02:21] "GET /metrics HTTP/1.1" 200 196295 "" "Prometheus/2.33.4" 2024-04-02T15:02:22.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:21 smithi169 ceph-mon[25801]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-02T15:02:22.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:21 smithi169 ceph-mon[25801]: Reconfiguring daemon alertmanager.a on smithi042 2024-04-02T15:02:22.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:21 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-02T15:02:22.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:21 smithi169 ceph-mon[25801]: osdmap e71: 8 total, 8 up, 8 in 2024-04-02T15:02:22.477 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:02:22 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:02:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:02:23.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:22 smithi042 ceph-mon[24805]: pgmap v61: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 219 B/s rd, 438 B/s wr, 1 op/s 2024-04-02T15:02:23.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:22 smithi042 ceph-mon[29473]: pgmap v61: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 219 B/s rd, 438 B/s wr, 1 op/s 2024-04-02T15:02:23.227 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:22 smithi169 ceph-mon[25801]: pgmap v61: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 219 B/s rd, 438 B/s wr, 1 op/s 2024-04-02T15:02:24.293 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:24 smithi042 systemd[1]: Stopping Ceph alertmanager.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:02:25.662 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:25 smithi042 ceph-mon[24805]: pgmap v62: 129 pgs: 129 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 1 op/s 2024-04-02T15:02:25.662 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:25 smithi042 ceph-mon[29473]: pgmap v62: 129 pgs: 129 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 1 op/s 2024-04-02T15:02:25.835 INFO:teuthology.orchestra.run.smithi042.stdout:Scheduled rgw.foo update... 2024-04-02T15:02:25.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:25 smithi169 ceph-mon[25801]: pgmap v62: 129 pgs: 129 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 1 op/s 2024-04-02T15:02:26.042 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:25 smithi042 bash[49279]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-02T15:02:26.043 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:25 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[47630]: level=info ts=2024-04-02T15:02:25.788Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-02T15:02:26.043 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:25 smithi042 podman[49358]: 2024-04-02 15:02:25.809210067 +0000 UTC m=+0.125656359 container died 53015fa2ce81b4c0e82653aaa7489a9ba8add5b3c895c24a70fc2a14d12f6ede (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:02:26.506 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:26 smithi042 podman[49358]: 2024-04-02 15:02:26.241346381 +0000 UTC m=+0.557792676 container cleanup 53015fa2ce81b4c0e82653aaa7489a9ba8add5b3c895c24a70fc2a14d12f6ede (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:02:26.506 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:26 smithi042 bash[49358]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a 2024-04-02T15:02:26.506 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:26 smithi042 podman[49372]: 2024-04-02 15:02:26.459224833 +0000 UTC m=+0.646605703 container remove 53015fa2ce81b4c0e82653aaa7489a9ba8add5b3c895c24a70fc2a14d12f6ede (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:02:26.506 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:26 smithi042 podman[49372]: 2024-04-02 15:02:26.518254979 +0000 UTC m=+0.705635848 volume remove 8682429ba6b830837dd6a60013c10a6e05df15ff6fb129fae9285544228b1a77 2024-04-02T15:02:26.793 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:26 smithi042 bash[49397]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-02T15:02:26.793 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:26 smithi042 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-02T15:02:26.793 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:26 smithi042 systemd[1]: Stopped Ceph alertmanager.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T15:02:26.793 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:26 smithi042 systemd[1]: Starting Ceph alertmanager.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:02:27.227 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:26 smithi169 ceph-mon[25801]: from='client.24610 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:02:27.227 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:26 smithi169 ceph-mon[25801]: Saving service rgw.foo spec with placement count:2 2024-04-02T15:02:27.227 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:26 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:27.227 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:26 smithi169 ceph-mon[25801]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-04-02T15:02:27.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:26 smithi042 ceph-mon[24805]: from='client.24610 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:02:27.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:26 smithi042 ceph-mon[24805]: Saving service rgw.foo spec with placement count:2 2024-04-02T15:02:27.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:26 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:27.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:26 smithi042 ceph-mon[24805]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-04-02T15:02:27.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:26 smithi042 ceph-mon[29473]: from='client.24610 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:02:27.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:26 smithi042 ceph-mon[29473]: Saving service rgw.foo spec with placement count:2 2024-04-02T15:02:27.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:26 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:27.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:26 smithi042 ceph-mon[29473]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-04-02T15:02:27.334 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph osd pool create foo' 2024-04-02T15:02:27.645 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:27 smithi042 podman[49485]: 2024-04-02 15:02:27.540065167 +0000 UTC m=+0.113710034 volume create a7855cf7da939d769415e8c0de2fbac5e8c9557ca79fcbef35918abcc8bceae0 2024-04-02T15:02:27.645 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:27 smithi042 podman[49485]: 2024-04-02 15:02:27.445160089 +0000 UTC m=+0.018804967 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-02T15:02:27.645 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:27 smithi042 podman[49485]: 2024-04-02 15:02:27.657269954 +0000 UTC m=+0.230914820 container create 64e49f76da9a8efc5889cbc6cf70d6cf7eccab9993d792ef34ae0667b5b31391 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:02:28.293 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:28 smithi042 podman[49485]: 2024-04-02 15:02:28.005503403 +0000 UTC m=+0.579148285 container init 64e49f76da9a8efc5889cbc6cf70d6cf7eccab9993d792ef34ae0667b5b31391 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:02:28.293 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:28 smithi042 podman[49485]: 2024-04-02 15:02:28.008873503 +0000 UTC m=+0.582518373 container start 64e49f76da9a8efc5889cbc6cf70d6cf7eccab9993d792ef34ae0667b5b31391 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:02:28.293 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=info ts=2024-04-02T15:02:28.019Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-02T15:02:28.293 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=info ts=2024-04-02T15:02:28.019Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-02T15:02:28.293 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=info ts=2024-04-02T15:02:28.020Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.42 port=9094 2024-04-02T15:02:28.293 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=info ts=2024-04-02T15:02:28.021Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-02T15:02:28.293 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=info ts=2024-04-02T15:02:28.046Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-02T15:02:28.293 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=info ts=2024-04-02T15:02:28.046Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-02T15:02:28.293 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=info ts=2024-04-02T15:02:28.048Z caller=main.go:518 msg=Listening address=:9093 2024-04-02T15:02:28.293 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=info ts=2024-04-02T15:02:28.048Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-02T15:02:28.293 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:28 smithi042 bash[49485]: 64e49f76da9a8efc5889cbc6cf70d6cf7eccab9993d792ef34ae0667b5b31391 2024-04-02T15:02:28.294 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:28 smithi042 systemd[1]: Started Ceph alertmanager.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T15:02:29.391 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:29 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:29.391 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:29 smithi042 ceph-mon[24805]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T15:02:29.391 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:29 smithi042 ceph-mon[24805]: Reconfiguring daemon prometheus.a on smithi169 2024-04-02T15:02:29.391 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:29 smithi042 ceph-mon[24805]: pgmap v64: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 827 B/s wr, 2 op/s 2024-04-02T15:02:29.391 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:29 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:29.392 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:29 smithi042 ceph-mon[29473]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T15:02:29.392 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:29 smithi042 ceph-mon[29473]: Reconfiguring daemon prometheus.a on smithi169 2024-04-02T15:02:29.392 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:29 smithi042 ceph-mon[29473]: pgmap v64: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 827 B/s wr, 2 op/s 2024-04-02T15:02:29.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:29 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:29.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:29 smithi169 ceph-mon[25801]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T15:02:29.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:29 smithi169 ceph-mon[25801]: Reconfiguring daemon prometheus.a on smithi169 2024-04-02T15:02:29.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:29 smithi169 ceph-mon[25801]: pgmap v64: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 827 B/s wr, 2 op/s 2024-04-02T15:02:30.148 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'foo' created 2024-04-02T15:02:30.292 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:30 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3829146163' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T15:02:30.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:30 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T15:02:30.293 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:30 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3829146163' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T15:02:30.293 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:30 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T15:02:30.293 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:30 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=info ts=2024-04-02T15:02:30.021Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000176407s 2024-04-02T15:02:30.449 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:30 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3829146163' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T15:02:30.449 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:30 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T15:02:30.843 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'rbd pool init foo' 2024-04-02T15:02:31.340 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:31 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-02T15:02:31.340 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:31 smithi169 ceph-mon[25801]: osdmap e72: 8 total, 8 up, 8 in 2024-04-02T15:02:31.340 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:31 smithi169 ceph-mon[25801]: pgmap v66: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 742 B/s rd, 742 B/s wr, 1 op/s 2024-04-02T15:02:31.340 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 systemd[1]: Stopping Ceph prometheus.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:02:31.340 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 bash[42596]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T15:02:31.473 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:31 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-02T15:02:31.473 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:31 smithi042 ceph-mon[24805]: osdmap e72: 8 total, 8 up, 8 in 2024-04-02T15:02:31.473 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:31 smithi042 ceph-mon[24805]: pgmap v66: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 742 B/s rd, 742 B/s wr, 1 op/s 2024-04-02T15:02:31.474 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:31 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-02T15:02:31.474 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:31 smithi042 ceph-mon[29473]: osdmap e72: 8 total, 8 up, 8 in 2024-04-02T15:02:31.474 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:31 smithi042 ceph-mon[29473]: pgmap v66: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 742 B/s rd, 742 B/s wr, 1 op/s 2024-04-02T15:02:31.641 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:02:31.351Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-02T15:02:31.641 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:02:31.352Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-02T15:02:31.641 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:02:31.352Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-02T15:02:31.641 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:02:31.352Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-02T15:02:31.641 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:02:31.352Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-02T15:02:31.641 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:02:31.352Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-02T15:02:31.641 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:02:31.352Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-02T15:02:31.641 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:02:31.352Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-02T15:02:31.641 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:02:31.352Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-02T15:02:31.642 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:02:31.352Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-02T15:02:31.642 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:02:31.352Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-02T15:02:31.642 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[40945]: ts=2024-04-02T15:02:31.352Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-02T15:02:31.642 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 podman[42609]: 2024-04-02 15:02:31.361715777 +0000 UTC m=+0.109836377 container died 44d2bff3219178a41210ffaa2cc68b596cf1e8aac20725b55a8668c9bf0edbc0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:02:31.977 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 podman[42609]: 2024-04-02 15:02:31.751517188 +0000 UTC m=+0.499637786 container cleanup 44d2bff3219178a41210ffaa2cc68b596cf1e8aac20725b55a8668c9bf0edbc0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:02:31.977 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 bash[42609]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a 2024-04-02T15:02:31.977 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 bash[42635]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T15:02:32.258 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-mon[25801]: osdmap e73: 8 total, 8 up, 8 in 2024-04-02T15:02:32.259 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 podman[42623]: 2024-04-02 15:02:31.994405558 +0000 UTC m=+0.634647054 container remove 44d2bff3219178a41210ffaa2cc68b596cf1e8aac20725b55a8668c9bf0edbc0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:02:32.259 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-02T15:02:32.259 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:31 smithi169 systemd[1]: Stopped Ceph prometheus.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T15:02:32.259 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 systemd[1]: Starting Ceph prometheus.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:02:32.259 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 podman[42702]: 2024-04-02 15:02:32.138047641 +0000 UTC m=+0.019064698 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-02T15:02:32.259 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 podman[42702]: 2024-04-02 15:02:32.270396971 +0000 UTC m=+0.151414018 container create 80b1a5d12473b3ca01a442b3365bd5dbbbacb1b0b313b1fb8a7a9d4e1924c230 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:02:32.447 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:32 smithi042 ceph-mon[24805]: osdmap e73: 8 total, 8 up, 8 in 2024-04-02T15:02:32.448 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:32 smithi042 ceph-mon[29473]: osdmap e73: 8 total, 8 up, 8 in 2024-04-02T15:02:32.774 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 podman[42702]: 2024-04-02 15:02:32.531048049 +0000 UTC m=+0.412065118 container init 80b1a5d12473b3ca01a442b3365bd5dbbbacb1b0b313b1fb8a7a9d4e1924c230 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:02:32.774 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 podman[42702]: 2024-04-02 15:02:32.533613019 +0000 UTC m=+0.414630076 container start 80b1a5d12473b3ca01a442b3365bd5dbbbacb1b0b313b1fb8a7a9d4e1924c230 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:02:32.774 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.556Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-02T15:02:32.774 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.556Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-02T15:02:32.775 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.556Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-02T15:02:32.775 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.556Z caller=main.go:518 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi169 (none))" 2024-04-02T15:02:32.775 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.556Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-02T15:02:32.775 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.556Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-02T15:02:32.775 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.558Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-02T15:02:32.775 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.559Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-02T15:02:32.775 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.560Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-02T15:02:32.775 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.562Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-02T15:02:32.775 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.562Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.54µs 2024-04-02T15:02:32.775 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.562Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-02T15:02:32.776 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 bash[42702]: 80b1a5d12473b3ca01a442b3365bd5dbbbacb1b0b313b1fb8a7a9d4e1924c230 2024-04-02T15:02:32.776 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 systemd[1]: Started Ceph prometheus.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T15:02:32.776 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.784Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-02T15:02:32.776 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.784Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-02T15:02:32.776 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.785Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.268µs wal_replay_duration=222.716675ms total_replay_duration=222.757592ms 2024-04-02T15:02:33.161 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.788Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-02T15:02:33.161 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.788Z caller=main.go:947 level=info msg="TSDB started" 2024-04-02T15:02:33.161 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.788Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-02T15:02:33.161 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.803Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.654978ms db_storage=795ns remote_storage=1.189µs web_handler=277ns query_engine=1.091µs scrape=519.18µs scrape_sd=35.779µs notify=30.947µs notify_sd=12.839µs rules=13.726462ms 2024-04-02T15:02:33.161 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:02:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:02:32.803Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-02T15:02:33.429 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: osdmap e74: 8 total, 8 up, 8 in 2024-04-02T15:02:33.429 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:33.429 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:33.429 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T15:02:33.429 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.42:9093"}]: dispatch 2024-04-02T15:02:33.429 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:33.429 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:02:33.429 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.169:9095"}]: dispatch 2024-04-02T15:02:33.430 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:33.430 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T15:02:33.430 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.169:3000"}]: dispatch 2024-04-02T15:02:33.430 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:33.430 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:02:33.430 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:33.430 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:02:33.430 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1333581534' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-02T15:02:33.430 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:33 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-02T15:02:33.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: osdmap e74: 8 total, 8 up, 8 in 2024-04-02T15:02:33.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:33.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:33.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T15:02:33.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.42:9093"}]: dispatch 2024-04-02T15:02:33.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:33.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:02:33.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.169:9095"}]: dispatch 2024-04-02T15:02:33.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:33.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T15:02:33.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.169:3000"}]: dispatch 2024-04-02T15:02:33.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:33.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:02:33.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:33.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:02:33.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1333581534' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-02T15:02:33.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-02T15:02:33.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: osdmap e74: 8 total, 8 up, 8 in 2024-04-02T15:02:33.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:33.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:33.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T15:02:33.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.42:9093"}]: dispatch 2024-04-02T15:02:33.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:33.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:02:33.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.169:9095"}]: dispatch 2024-04-02T15:02:33.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:33.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T15:02:33.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.169:3000"}]: dispatch 2024-04-02T15:02:33.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:33.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:02:33.545 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:33.546 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:02:33.546 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1333581534' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-02T15:02:33.546 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:33 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-02T15:02:34.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:34 smithi169 ceph-mon[25801]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T15:02:34.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:34 smithi169 ceph-mon[25801]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.42:9093"}]: dispatch 2024-04-02T15:02:34.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:34 smithi169 ceph-mon[25801]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:02:34.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:34 smithi169 ceph-mon[25801]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.169:9095"}]: dispatch 2024-04-02T15:02:34.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:34 smithi169 ceph-mon[25801]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T15:02:34.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:34 smithi169 ceph-mon[25801]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.169:3000"}]: dispatch 2024-04-02T15:02:34.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:34 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-02T15:02:34.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:34 smithi169 ceph-mon[25801]: osdmap e75: 8 total, 8 up, 8 in 2024-04-02T15:02:34.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[24805]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T15:02:34.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[24805]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.42:9093"}]: dispatch 2024-04-02T15:02:34.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[24805]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:02:34.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[24805]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.169:9095"}]: dispatch 2024-04-02T15:02:34.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[24805]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T15:02:34.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[24805]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.169:3000"}]: dispatch 2024-04-02T15:02:34.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-02T15:02:34.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[24805]: osdmap e75: 8 total, 8 up, 8 in 2024-04-02T15:02:34.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[29473]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T15:02:34.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[29473]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.42:9093"}]: dispatch 2024-04-02T15:02:34.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[29473]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:02:34.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[29473]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.169:9095"}]: dispatch 2024-04-02T15:02:34.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[29473]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T15:02:34.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[29473]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.169:3000"}]: dispatch 2024-04-02T15:02:34.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-02T15:02:34.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:34 smithi042 ceph-mon[29473]: osdmap e75: 8 total, 8 up, 8 in 2024-04-02T15:02:35.472 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:35 smithi169 ceph-mon[25801]: pgmap v71: 161 pgs: 11 unknown, 150 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:35.472 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:35 smithi169 ceph-mon[25801]: osdmap e76: 8 total, 8 up, 8 in 2024-04-02T15:02:35.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:35 smithi042 ceph-mon[24805]: pgmap v71: 161 pgs: 11 unknown, 150 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:35.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:35 smithi042 ceph-mon[24805]: osdmap e76: 8 total, 8 up, 8 in 2024-04-02T15:02:35.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:35 smithi042 ceph-mon[29473]: pgmap v71: 161 pgs: 11 unknown, 150 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:35.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:35 smithi042 ceph-mon[29473]: osdmap e76: 8 total, 8 up, 8 in 2024-04-02T15:02:36.452 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch apply iscsi foo u p' 2024-04-02T15:02:36.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[29473]: osdmap e77: 8 total, 8 up, 8 in 2024-04-02T15:02:36.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi042.plghvb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:02:36.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi042.plghvb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:02:36.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi042.plghvb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T15:02:36.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:36.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[24805]: osdmap e77: 8 total, 8 up, 8 in 2024-04-02T15:02:36.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.544 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi042.plghvb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:02:36.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi042.plghvb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:02:36.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi042.plghvb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T15:02:36.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:36 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:36.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:36 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:36 smithi169 ceph-mon[25801]: osdmap e77: 8 total, 8 up, 8 in 2024-04-02T15:02:36.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:36 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:36 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:36 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:36 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:36 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:36 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi042.plghvb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:02:36.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:36 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi042.plghvb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:02:36.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:36 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi042.plghvb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T15:02:36.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:36 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:36.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:36 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:37.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:37 smithi042 ceph-mon[29473]: Saving service rgw.foo spec with placement count:2 2024-04-02T15:02:37.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:37 smithi042 ceph-mon[29473]: Deploying daemon rgw.foo.smithi042.plghvb on smithi042 2024-04-02T15:02:37.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:37 smithi042 ceph-mon[29473]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:37.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:37 smithi042 ceph-mon[24805]: Saving service rgw.foo spec with placement count:2 2024-04-02T15:02:37.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:37 smithi042 ceph-mon[24805]: Deploying daemon rgw.foo.smithi042.plghvb on smithi042 2024-04-02T15:02:37.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:37 smithi042 ceph-mon[24805]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:37.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:37 smithi169 ceph-mon[25801]: Saving service rgw.foo spec with placement count:2 2024-04-02T15:02:37.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:37 smithi169 ceph-mon[25801]: Deploying daemon rgw.foo.smithi042.plghvb on smithi042 2024-04-02T15:02:37.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:37 smithi169 ceph-mon[25801]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:38.283 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:02:38 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=info ts=2024-04-02T15:02:38.023Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.001934916s 2024-04-02T15:02:39.645 INFO:teuthology.orchestra.run.smithi042.stdout:Scheduled iscsi.foo update... 2024-04-02T15:02:39.751 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:39 smithi042 ceph-mon[24805]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:39.751 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:39 smithi042 ceph-mon[29473]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:39.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:39 smithi169 ceph-mon[25801]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:02:40.698 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[29473]: from='client.14706 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:02:40.698 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[29473]: Saving service iscsi.foo spec with placement count:1 2024-04-02T15:02:40.699 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:40.699 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:40.699 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi169.pvuzch", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:02:40.699 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi169.pvuzch", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:02:40.699 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi169.pvuzch", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T15:02:40.699 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:40.699 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:40.699 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[29473]: Deploying daemon rgw.foo.smithi169.pvuzch on smithi169 2024-04-02T15:02:40.699 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[29473]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 303 B/s wr, 0 op/s 2024-04-02T15:02:40.699 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[24805]: from='client.14706 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:02:40.699 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[24805]: Saving service iscsi.foo spec with placement count:1 2024-04-02T15:02:40.699 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:40.700 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:40.700 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi169.pvuzch", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:02:40.700 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi169.pvuzch", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:02:40.700 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi169.pvuzch", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T15:02:40.700 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:40.700 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:40.700 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[24805]: Deploying daemon rgw.foo.smithi169.pvuzch on smithi169 2024-04-02T15:02:40.701 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:40 smithi042 ceph-mon[24805]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 303 B/s wr, 0 op/s 2024-04-02T15:02:40.721 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'sleep 180' 2024-04-02T15:02:40.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:40 smithi169 ceph-mon[25801]: from='client.14706 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:02:40.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:40 smithi169 ceph-mon[25801]: Saving service iscsi.foo spec with placement count:1 2024-04-02T15:02:40.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:40 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:40.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:40 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:40.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:40 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi169.pvuzch", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:02:40.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:40 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi169.pvuzch", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:02:40.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:40 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi169.pvuzch", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T15:02:40.895 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:40 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:40.896 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:40 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:40.896 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:40 smithi169 ceph-mon[25801]: Deploying daemon rgw.foo.smithi169.pvuzch on smithi169 2024-04-02T15:02:40.896 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:40 smithi169 ceph-mon[25801]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 303 B/s wr, 0 op/s 2024-04-02T15:02:42.043 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:02:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:02:41] "GET /metrics HTTP/1.1" 200 197838 "" "Prometheus/2.33.4" 2024-04-02T15:02:42.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:02:42 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:02:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:02:43.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:43 smithi042 ceph-mon[24805]: pgmap v77: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-02T15:02:43.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:43 smithi042 ceph-mon[29473]: pgmap v77: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-02T15:02:43.801 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:43 smithi169 ceph-mon[25801]: pgmap v77: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-02T15:02:45.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:45 smithi042 ceph-mon[24805]: pgmap v78: 161 pgs: 161 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 2.9 KiB/s wr, 81 op/s 2024-04-02T15:02:45.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:45 smithi042 ceph-mon[24805]: osdmap e78: 8 total, 8 up, 8 in 2024-04-02T15:02:45.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:45 smithi042 ceph-mon[29473]: pgmap v78: 161 pgs: 161 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 2.9 KiB/s wr, 81 op/s 2024-04-02T15:02:45.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:45 smithi042 ceph-mon[29473]: osdmap e78: 8 total, 8 up, 8 in 2024-04-02T15:02:45.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:45 smithi169 ceph-mon[25801]: pgmap v78: 161 pgs: 161 active+clean; 583 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 2.9 KiB/s wr, 81 op/s 2024-04-02T15:02:45.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:45 smithi169 ceph-mon[25801]: osdmap e78: 8 total, 8 up, 8 in 2024-04-02T15:02:47.547 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:47 smithi169 ceph-mon[25801]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 112 op/s 2024-04-02T15:02:47.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:47 smithi042 ceph-mon[24805]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 112 op/s 2024-04-02T15:02:47.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:47 smithi042 ceph-mon[29473]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 112 op/s 2024-04-02T15:02:48.673 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:48.673 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:02:48.673 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:48.673 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:02:48.674 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]: dispatch 2024-04-02T15:02:48.674 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-02T15:02:48.674 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]: dispatch 2024-04-02T15:02:48.674 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-04-02T15:02:48.674 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-02T15:02:48.674 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-02T15:02:48.674 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-04-02T15:02:48.674 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-02T15:02:48.674 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 112 op/s 2024-04-02T15:02:48.674 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:02:48.674 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:02:48.675 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:48.675 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:02:48.675 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:02:49.022 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:49.022 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:02:49.022 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:49.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:02:49.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]: dispatch 2024-04-02T15:02:49.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-02T15:02:49.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]: dispatch 2024-04-02T15:02:49.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-04-02T15:02:49.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-02T15:02:49.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-02T15:02:49.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-04-02T15:02:49.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-02T15:02:49.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 112 op/s 2024-04-02T15:02:49.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:02:49.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:02:49.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:49.024 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:02:49.024 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:02:49.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:49.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:02:49.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:49.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:02:49.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]: dispatch 2024-04-02T15:02:49.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-02T15:02:49.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]: dispatch 2024-04-02T15:02:49.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-04-02T15:02:49.025 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-02T15:02:49.025 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-02T15:02:49.025 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-04-02T15:02:49.025 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-02T15:02:49.025 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 112 op/s 2024-04-02T15:02:49.025 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:02:49.025 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:02:49.025 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:49.025 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:02:49.025 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:02:49.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:49 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]': finished 2024-04-02T15:02:49.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:49 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-02T15:02:49.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:49 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-04-02T15:02:49.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:49 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-02T15:02:49.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:49 smithi169 ceph-mon[25801]: osdmap e79: 8 total, 8 up, 8 in 2024-04-02T15:02:49.845 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:49 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]': finished 2024-04-02T15:02:49.845 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:49 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-02T15:02:49.845 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:49 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-04-02T15:02:49.845 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:49 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-02T15:02:49.845 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:49 smithi042 ceph-mon[29473]: osdmap e79: 8 total, 8 up, 8 in 2024-04-02T15:02:49.846 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:49 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]': finished 2024-04-02T15:02:49.846 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:49 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-02T15:02:49.846 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:49 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-04-02T15:02:49.846 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:49 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-02T15:02:49.846 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:49 smithi042 ceph-mon[24805]: osdmap e79: 8 total, 8 up, 8 in 2024-04-02T15:02:50.720 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:50 smithi169 ceph-mon[25801]: osdmap e80: 8 total, 8 up, 8 in 2024-04-02T15:02:50.720 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:50 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:50.720 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:50 smithi169 ceph-mon[25801]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 1.3 KiB/s wr, 220 op/s 2024-04-02T15:02:51.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:50 smithi042 ceph-mon[24805]: osdmap e80: 8 total, 8 up, 8 in 2024-04-02T15:02:51.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:50 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:51.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:50 smithi042 ceph-mon[24805]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 1.3 KiB/s wr, 220 op/s 2024-04-02T15:02:51.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:50 smithi042 ceph-mon[29473]: osdmap e80: 8 total, 8 up, 8 in 2024-04-02T15:02:51.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:50 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:51.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:50 smithi042 ceph-mon[29473]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 1.3 KiB/s wr, 220 op/s 2024-04-02T15:02:51.993 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:02:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:02:51] "GET /metrics HTTP/1.1" 200 207190 "" "Prometheus/2.33.4" 2024-04-02T15:02:52.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:52.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[24805]: Checking dashboard <-> RGW credentials 2024-04-02T15:02:52.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:52.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:52.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:52.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[24805]: Checking pool "foo" exists for service iscsi.foo 2024-04-02T15:02:52.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi042.toyfgf", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T15:02:52.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi042.toyfgf", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T15:02:52.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi042.toyfgf", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-02T15:02:52.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:52.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[24805]: Deploying daemon iscsi.foo.smithi042.toyfgf on smithi042 2024-04-02T15:02:52.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:52.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[29473]: Checking dashboard <-> RGW credentials 2024-04-02T15:02:52.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:52.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:52.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:52.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[29473]: Checking pool "foo" exists for service iscsi.foo 2024-04-02T15:02:52.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi042.toyfgf", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T15:02:52.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi042.toyfgf", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T15:02:52.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi042.toyfgf", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-02T15:02:52.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:52.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:52 smithi042 ceph-mon[29473]: Deploying daemon iscsi.foo.smithi042.toyfgf on smithi042 2024-04-02T15:02:52.477 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:02:52 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:02:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:02:52.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:52 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:52.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:52 smithi169 ceph-mon[25801]: Checking dashboard <-> RGW credentials 2024-04-02T15:02:52.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:52 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:52.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:52 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:52.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:52 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:52.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:52 smithi169 ceph-mon[25801]: Checking pool "foo" exists for service iscsi.foo 2024-04-02T15:02:52.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:52 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi042.toyfgf", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T15:02:52.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:52 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi042.toyfgf", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T15:02:52.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:52 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi042.toyfgf", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-02T15:02:52.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:52 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:52.478 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:52 smithi169 ceph-mon[25801]: Deploying daemon iscsi.foo.smithi042.toyfgf on smithi042 2024-04-02T15:02:53.663 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:53 smithi042 ceph-mon[24805]: mgrmap e21: y(active, since 2m), standbys: x 2024-04-02T15:02:53.663 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:53 smithi042 ceph-mon[24805]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 142 B/s wr, 139 op/s 2024-04-02T15:02:53.664 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:53 smithi042 ceph-mon[29473]: mgrmap e21: y(active, since 2m), standbys: x 2024-04-02T15:02:53.664 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:53 smithi042 ceph-mon[29473]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 142 B/s wr, 139 op/s 2024-04-02T15:02:53.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:53 smithi169 ceph-mon[25801]: mgrmap e21: y(active, since 2m), standbys: x 2024-04-02T15:02:53.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:53 smithi169 ceph-mon[25801]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 142 B/s wr, 139 op/s 2024-04-02T15:02:55.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:55 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:55.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:55 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:02:55.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:55 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:55.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:55 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:02:55.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:55 smithi169 ceph-mon[25801]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 511 B/s wr, 130 op/s; 5 B/s, 0 objects/s recovering 2024-04-02T15:02:55.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:55 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/2821531270' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T15:02:55.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:55 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1154462136' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/2635164233"}]: dispatch 2024-04-02T15:02:55.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:55 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:55.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:55 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:02:55.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:55 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:55.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:55 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:02:55.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:55 smithi042 ceph-mon[24805]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 511 B/s wr, 130 op/s; 5 B/s, 0 objects/s recovering 2024-04-02T15:02:55.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:55 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/2821531270' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T15:02:55.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:55 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1154462136' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/2635164233"}]: dispatch 2024-04-02T15:02:55.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:55 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:55.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:55 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:02:55.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:55 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:55.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:55 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:02:55.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:55 smithi042 ceph-mon[29473]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 511 B/s wr, 130 op/s; 5 B/s, 0 objects/s recovering 2024-04-02T15:02:55.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:55 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/2821531270' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T15:02:55.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:55 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1154462136' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/2635164233"}]: dispatch 2024-04-02T15:02:56.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:56 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1154462136' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/2635164233"}]': finished 2024-04-02T15:02:56.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:56 smithi042 ceph-mon[24805]: osdmap e81: 8 total, 8 up, 8 in 2024-04-02T15:02:56.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:56 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/2380772764' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/280964113"}]: dispatch 2024-04-02T15:02:56.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:56 smithi042 ceph-mon[24805]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/280964113"}]: dispatch 2024-04-02T15:02:56.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:56 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1154462136' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/2635164233"}]': finished 2024-04-02T15:02:56.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:56 smithi042 ceph-mon[29473]: osdmap e81: 8 total, 8 up, 8 in 2024-04-02T15:02:56.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:56 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/2380772764' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/280964113"}]: dispatch 2024-04-02T15:02:56.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:56 smithi042 ceph-mon[29473]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/280964113"}]: dispatch 2024-04-02T15:02:56.603 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:56 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1154462136' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/2635164233"}]': finished 2024-04-02T15:02:56.603 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:56 smithi169 ceph-mon[25801]: osdmap e81: 8 total, 8 up, 8 in 2024-04-02T15:02:56.603 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:56 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/2380772764' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/280964113"}]: dispatch 2024-04-02T15:02:56.603 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:56 smithi169 ceph-mon[25801]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/280964113"}]: dispatch 2024-04-02T15:02:57.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:57 smithi169 ceph-mon[25801]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/280964113"}]': finished 2024-04-02T15:02:57.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:57 smithi169 ceph-mon[25801]: osdmap e82: 8 total, 8 up, 8 in 2024-04-02T15:02:57.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:57 smithi169 ceph-mon[25801]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 6.3 KiB/s rd, 446 B/s wr, 7 op/s; 6 B/s, 0 objects/s recovering 2024-04-02T15:02:57.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:57 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1099240357' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3168280765"}]: dispatch 2024-04-02T15:02:57.477 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:57 smithi169 ceph-mon[25801]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3168280765"}]: dispatch 2024-04-02T15:02:57.634 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:57 smithi042 ceph-mon[24805]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/280964113"}]': finished 2024-04-02T15:02:57.634 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:57 smithi042 ceph-mon[24805]: osdmap e82: 8 total, 8 up, 8 in 2024-04-02T15:02:57.634 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:57 smithi042 ceph-mon[24805]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 6.3 KiB/s rd, 446 B/s wr, 7 op/s; 6 B/s, 0 objects/s recovering 2024-04-02T15:02:57.634 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:57 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1099240357' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3168280765"}]: dispatch 2024-04-02T15:02:57.635 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:57 smithi042 ceph-mon[24805]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3168280765"}]: dispatch 2024-04-02T15:02:57.635 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:57 smithi042 ceph-mon[29473]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/280964113"}]': finished 2024-04-02T15:02:57.635 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:57 smithi042 ceph-mon[29473]: osdmap e82: 8 total, 8 up, 8 in 2024-04-02T15:02:57.635 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:57 smithi042 ceph-mon[29473]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 6.3 KiB/s rd, 446 B/s wr, 7 op/s; 6 B/s, 0 objects/s recovering 2024-04-02T15:02:57.635 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:57 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1099240357' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3168280765"}]: dispatch 2024-04-02T15:02:57.635 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:57 smithi042 ceph-mon[29473]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3168280765"}]: dispatch 2024-04-02T15:02:58.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:58 smithi169 ceph-mon[25801]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3168280765"}]': finished 2024-04-02T15:02:58.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:58 smithi169 ceph-mon[25801]: osdmap e83: 8 total, 8 up, 8 in 2024-04-02T15:02:58.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:58 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3748095062' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/2888334500"}]: dispatch 2024-04-02T15:02:58.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:58 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:58 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:58 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:58 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T15:02:58.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:58 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T15:02:58.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:58 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:58 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi042"}]: dispatch 2024-04-02T15:02:58.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:58 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:58 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:02:58.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:58 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:58.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:58 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:02:58.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:58 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[29473]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3168280765"}]': finished 2024-04-02T15:02:58.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[29473]: osdmap e83: 8 total, 8 up, 8 in 2024-04-02T15:02:58.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3748095062' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/2888334500"}]: dispatch 2024-04-02T15:02:58.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T15:02:58.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T15:02:58.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi042"}]: dispatch 2024-04-02T15:02:58.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:02:58.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:58.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:02:58.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[24805]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3168280765"}]': finished 2024-04-02T15:02:58.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[24805]: osdmap e83: 8 total, 8 up, 8 in 2024-04-02T15:02:58.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3748095062' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/2888334500"}]: dispatch 2024-04-02T15:02:58.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T15:02:58.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T15:02:58.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi042"}]: dispatch 2024-04-02T15:02:58.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:58.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:02:58.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:02:58.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:02:58.795 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:58 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:59.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:59 smithi169 ceph-mon[25801]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T15:02:59.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:59 smithi169 ceph-mon[25801]: Adding iSCSI gateway http://:@172.21.15.42:5000 to Dashboard 2024-04-02T15:02:59.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:59 smithi169 ceph-mon[25801]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T15:02:59.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:59 smithi169 ceph-mon[25801]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi042"}]: dispatch 2024-04-02T15:02:59.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:59 smithi169 ceph-mon[25801]: Checking dashboard <-> RGW credentials 2024-04-02T15:02:59.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:59 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3748095062' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/2888334500"}]': finished 2024-04-02T15:02:59.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:59 smithi169 ceph-mon[25801]: osdmap e84: 8 total, 8 up, 8 in 2024-04-02T15:02:59.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:59 smithi169 ceph-mon[25801]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 3 op/s 2024-04-02T15:02:59.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:59 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:59.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:02:59 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3580562153' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1443281804"}]: dispatch 2024-04-02T15:02:59.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[24805]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T15:02:59.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[24805]: Adding iSCSI gateway http://:@172.21.15.42:5000 to Dashboard 2024-04-02T15:02:59.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[24805]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T15:02:59.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[24805]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi042"}]: dispatch 2024-04-02T15:02:59.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[24805]: Checking dashboard <-> RGW credentials 2024-04-02T15:02:59.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3748095062' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/2888334500"}]': finished 2024-04-02T15:02:59.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[24805]: osdmap e84: 8 total, 8 up, 8 in 2024-04-02T15:02:59.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[24805]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 3 op/s 2024-04-02T15:02:59.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:59.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3580562153' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1443281804"}]: dispatch 2024-04-02T15:02:59.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[29473]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T15:02:59.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[29473]: Adding iSCSI gateway http://:@172.21.15.42:5000 to Dashboard 2024-04-02T15:02:59.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[29473]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T15:02:59.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[29473]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi042"}]: dispatch 2024-04-02T15:02:59.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[29473]: Checking dashboard <-> RGW credentials 2024-04-02T15:02:59.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3748095062' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/2888334500"}]': finished 2024-04-02T15:02:59.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[29473]: osdmap e84: 8 total, 8 up, 8 in 2024-04-02T15:02:59.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[29473]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 3 op/s 2024-04-02T15:02:59.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:02:59.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:02:59 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3580562153' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1443281804"}]: dispatch 2024-04-02T15:03:00.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:00 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3580562153' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1443281804"}]': finished 2024-04-02T15:03:00.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:00 smithi042 ceph-mon[29473]: osdmap e85: 8 total, 8 up, 8 in 2024-04-02T15:03:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:00 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/4271145411' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/2350866342"}]: dispatch 2024-04-02T15:03:00.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:00 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3580562153' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1443281804"}]': finished 2024-04-02T15:03:00.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:00 smithi042 ceph-mon[24805]: osdmap e85: 8 total, 8 up, 8 in 2024-04-02T15:03:00.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:00 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/4271145411' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/2350866342"}]: dispatch 2024-04-02T15:03:00.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:00 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3580562153' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1443281804"}]': finished 2024-04-02T15:03:00.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:00 smithi169 ceph-mon[25801]: osdmap e85: 8 total, 8 up, 8 in 2024-04-02T15:03:00.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:00 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/4271145411' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/2350866342"}]: dispatch 2024-04-02T15:03:01.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:01 smithi042 ceph-mon[24805]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 8 op/s 2024-04-02T15:03:01.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:01 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/4271145411' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/2350866342"}]': finished 2024-04-02T15:03:01.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:01 smithi042 ceph-mon[24805]: osdmap e86: 8 total, 8 up, 8 in 2024-04-02T15:03:01.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:01 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1930030901' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3792138265"}]: dispatch 2024-04-02T15:03:01.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:01 smithi042 ceph-mon[24805]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3792138265"}]: dispatch 2024-04-02T15:03:01.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:01 smithi042 ceph-mon[29473]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 8 op/s 2024-04-02T15:03:01.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:01 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/4271145411' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/2350866342"}]': finished 2024-04-02T15:03:01.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:01 smithi042 ceph-mon[29473]: osdmap e86: 8 total, 8 up, 8 in 2024-04-02T15:03:01.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:01 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1930030901' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3792138265"}]: dispatch 2024-04-02T15:03:01.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:01 smithi042 ceph-mon[29473]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3792138265"}]: dispatch 2024-04-02T15:03:01.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:01 smithi169 ceph-mon[25801]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 8 op/s 2024-04-02T15:03:01.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:01 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/4271145411' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/2350866342"}]': finished 2024-04-02T15:03:01.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:01 smithi169 ceph-mon[25801]: osdmap e86: 8 total, 8 up, 8 in 2024-04-02T15:03:01.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:01 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1930030901' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3792138265"}]: dispatch 2024-04-02T15:03:01.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:01 smithi169 ceph-mon[25801]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3792138265"}]: dispatch 2024-04-02T15:03:02.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:03:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:03:01] "GET /metrics HTTP/1.1" 200 207190 "" "Prometheus/2.33.4" 2024-04-02T15:03:02.477 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:03:02 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:03:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:03:02.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:02 smithi042 ceph-mon[29473]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3792138265"}]': finished 2024-04-02T15:03:02.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:02 smithi042 ceph-mon[29473]: osdmap e87: 8 total, 8 up, 8 in 2024-04-02T15:03:02.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:02 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3895678340' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/1586110338"}]: dispatch 2024-04-02T15:03:02.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:02 smithi042 ceph-mon[29473]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/1586110338"}]: dispatch 2024-04-02T15:03:02.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:02 smithi042 ceph-mon[24805]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3792138265"}]': finished 2024-04-02T15:03:02.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:02 smithi042 ceph-mon[24805]: osdmap e87: 8 total, 8 up, 8 in 2024-04-02T15:03:02.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:02 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3895678340' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/1586110338"}]: dispatch 2024-04-02T15:03:02.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:02 smithi042 ceph-mon[24805]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/1586110338"}]: dispatch 2024-04-02T15:03:02.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:02 smithi169 ceph-mon[25801]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3792138265"}]': finished 2024-04-02T15:03:02.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:02 smithi169 ceph-mon[25801]: osdmap e87: 8 total, 8 up, 8 in 2024-04-02T15:03:02.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:02 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3895678340' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/1586110338"}]: dispatch 2024-04-02T15:03:02.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:02 smithi169 ceph-mon[25801]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/1586110338"}]: dispatch 2024-04-02T15:03:03.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:03 smithi042 ceph-mon[24805]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 8 op/s 2024-04-02T15:03:03.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:03 smithi042 ceph-mon[24805]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/1586110338"}]': finished 2024-04-02T15:03:03.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:03 smithi042 ceph-mon[24805]: osdmap e88: 8 total, 8 up, 8 in 2024-04-02T15:03:03.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:03 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1258666005' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/1586110338"}]: dispatch 2024-04-02T15:03:03.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:03 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:03:03.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:03 smithi042 ceph-mon[29473]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 8 op/s 2024-04-02T15:03:03.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:03 smithi042 ceph-mon[29473]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/1586110338"}]': finished 2024-04-02T15:03:03.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:03 smithi042 ceph-mon[29473]: osdmap e88: 8 total, 8 up, 8 in 2024-04-02T15:03:03.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:03 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1258666005' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/1586110338"}]: dispatch 2024-04-02T15:03:03.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:03 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:03:03.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:03 smithi169 ceph-mon[25801]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 8 op/s 2024-04-02T15:03:03.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:03 smithi169 ceph-mon[25801]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/1586110338"}]': finished 2024-04-02T15:03:03.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:03 smithi169 ceph-mon[25801]: osdmap e88: 8 total, 8 up, 8 in 2024-04-02T15:03:03.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:03 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1258666005' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/1586110338"}]: dispatch 2024-04-02T15:03:03.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:03 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:03:04.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:04 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1258666005' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/1586110338"}]': finished 2024-04-02T15:03:04.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:04 smithi042 ceph-mon[24805]: osdmap e89: 8 total, 8 up, 8 in 2024-04-02T15:03:04.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:04 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1108367609' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/2635164233"}]: dispatch 2024-04-02T15:03:04.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:04 smithi042 ceph-mon[24805]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:04.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:04 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:04.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:04 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1258666005' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/1586110338"}]': finished 2024-04-02T15:03:04.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:04 smithi042 ceph-mon[29473]: osdmap e89: 8 total, 8 up, 8 in 2024-04-02T15:03:04.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:04 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1108367609' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/2635164233"}]: dispatch 2024-04-02T15:03:04.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:04 smithi042 ceph-mon[29473]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:04.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:04 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:04.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:04 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1258666005' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/1586110338"}]': finished 2024-04-02T15:03:04.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:04 smithi169 ceph-mon[25801]: osdmap e89: 8 total, 8 up, 8 in 2024-04-02T15:03:04.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:04 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1108367609' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/2635164233"}]: dispatch 2024-04-02T15:03:04.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:04 smithi169 ceph-mon[25801]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:04.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:04 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:05.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:05 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1108367609' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/2635164233"}]': finished 2024-04-02T15:03:05.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:05 smithi042 ceph-mon[29473]: osdmap e90: 8 total, 8 up, 8 in 2024-04-02T15:03:05.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:05 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3855468896' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3226246413"}]: dispatch 2024-04-02T15:03:05.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:05 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1108367609' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/2635164233"}]': finished 2024-04-02T15:03:05.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:05 smithi042 ceph-mon[24805]: osdmap e90: 8 total, 8 up, 8 in 2024-04-02T15:03:05.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:05 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3855468896' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3226246413"}]: dispatch 2024-04-02T15:03:05.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:05 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1108367609' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/2635164233"}]': finished 2024-04-02T15:03:05.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:05 smithi169 ceph-mon[25801]: osdmap e90: 8 total, 8 up, 8 in 2024-04-02T15:03:05.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:05 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3855468896' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3226246413"}]: dispatch 2024-04-02T15:03:06.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:06 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3855468896' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3226246413"}]': finished 2024-04-02T15:03:06.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:06 smithi169 ceph-mon[25801]: osdmap e91: 8 total, 8 up, 8 in 2024-04-02T15:03:06.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:06 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1120448612' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1870022407"}]: dispatch 2024-04-02T15:03:06.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:06 smithi169 ceph-mon[25801]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:07.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:06 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3855468896' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3226246413"}]': finished 2024-04-02T15:03:07.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:06 smithi042 ceph-mon[24805]: osdmap e91: 8 total, 8 up, 8 in 2024-04-02T15:03:07.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:06 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1120448612' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1870022407"}]: dispatch 2024-04-02T15:03:07.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:06 smithi042 ceph-mon[24805]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:07.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:06 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3855468896' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3226246413"}]': finished 2024-04-02T15:03:07.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:06 smithi042 ceph-mon[29473]: osdmap e91: 8 total, 8 up, 8 in 2024-04-02T15:03:07.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:06 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1120448612' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1870022407"}]: dispatch 2024-04-02T15:03:07.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:06 smithi042 ceph-mon[29473]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:07.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:07 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1120448612' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1870022407"}]': finished 2024-04-02T15:03:07.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:07 smithi169 ceph-mon[25801]: osdmap e92: 8 total, 8 up, 8 in 2024-04-02T15:03:07.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:07 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1075671139' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3766045960"}]: dispatch 2024-04-02T15:03:07.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:07 smithi169 ceph-mon[25801]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3766045960"}]: dispatch 2024-04-02T15:03:08.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:07 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1120448612' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1870022407"}]': finished 2024-04-02T15:03:08.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:07 smithi042 ceph-mon[24805]: osdmap e92: 8 total, 8 up, 8 in 2024-04-02T15:03:08.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:07 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1075671139' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3766045960"}]: dispatch 2024-04-02T15:03:08.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:07 smithi042 ceph-mon[24805]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3766045960"}]: dispatch 2024-04-02T15:03:08.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:07 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1120448612' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1870022407"}]': finished 2024-04-02T15:03:08.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:07 smithi042 ceph-mon[29473]: osdmap e92: 8 total, 8 up, 8 in 2024-04-02T15:03:08.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:07 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1075671139' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3766045960"}]: dispatch 2024-04-02T15:03:08.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:07 smithi042 ceph-mon[29473]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3766045960"}]: dispatch 2024-04-02T15:03:08.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:08 smithi169 ceph-mon[25801]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3766045960"}]': finished 2024-04-02T15:03:08.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:08 smithi169 ceph-mon[25801]: osdmap e93: 8 total, 8 up, 8 in 2024-04-02T15:03:08.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:08 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/993734606' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1412464921"}]: dispatch 2024-04-02T15:03:08.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:08 smithi169 ceph-mon[25801]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1412464921"}]: dispatch 2024-04-02T15:03:08.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:08 smithi169 ceph-mon[25801]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:03:09.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:08 smithi042 ceph-mon[24805]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3766045960"}]': finished 2024-04-02T15:03:09.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:08 smithi042 ceph-mon[24805]: osdmap e93: 8 total, 8 up, 8 in 2024-04-02T15:03:09.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:08 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/993734606' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1412464921"}]: dispatch 2024-04-02T15:03:09.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:08 smithi042 ceph-mon[24805]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1412464921"}]: dispatch 2024-04-02T15:03:09.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:08 smithi042 ceph-mon[24805]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:03:09.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:08 smithi042 ceph-mon[29473]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/3766045960"}]': finished 2024-04-02T15:03:09.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:08 smithi042 ceph-mon[29473]: osdmap e93: 8 total, 8 up, 8 in 2024-04-02T15:03:09.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:08 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/993734606' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1412464921"}]: dispatch 2024-04-02T15:03:09.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:08 smithi042 ceph-mon[29473]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1412464921"}]: dispatch 2024-04-02T15:03:09.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:08 smithi042 ceph-mon[29473]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:03:09.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:09 smithi169 ceph-mon[25801]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1412464921"}]': finished 2024-04-02T15:03:09.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:09 smithi169 ceph-mon[25801]: osdmap e94: 8 total, 8 up, 8 in 2024-04-02T15:03:09.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:09 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/2067251218' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/3054125886"}]: dispatch 2024-04-02T15:03:09.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:09 smithi169 ceph-mon[25801]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/3054125886"}]: dispatch 2024-04-02T15:03:10.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:09 smithi042 ceph-mon[24805]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1412464921"}]': finished 2024-04-02T15:03:10.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:09 smithi042 ceph-mon[24805]: osdmap e94: 8 total, 8 up, 8 in 2024-04-02T15:03:10.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:09 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/2067251218' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/3054125886"}]: dispatch 2024-04-02T15:03:10.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:09 smithi042 ceph-mon[24805]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/3054125886"}]: dispatch 2024-04-02T15:03:10.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:09 smithi042 ceph-mon[29473]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:0/1412464921"}]': finished 2024-04-02T15:03:10.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:09 smithi042 ceph-mon[29473]: osdmap e94: 8 total, 8 up, 8 in 2024-04-02T15:03:10.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:09 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/2067251218' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/3054125886"}]: dispatch 2024-04-02T15:03:10.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:09 smithi042 ceph-mon[29473]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/3054125886"}]: dispatch 2024-04-02T15:03:10.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:10 smithi169 ceph-mon[25801]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/3054125886"}]': finished 2024-04-02T15:03:10.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:10 smithi169 ceph-mon[25801]: osdmap e95: 8 total, 8 up, 8 in 2024-04-02T15:03:10.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:10 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/198292946' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/3054125886"}]: dispatch 2024-04-02T15:03:10.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:10 smithi169 ceph-mon[25801]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:11.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:10 smithi042 ceph-mon[24805]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/3054125886"}]': finished 2024-04-02T15:03:11.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:10 smithi042 ceph-mon[24805]: osdmap e95: 8 total, 8 up, 8 in 2024-04-02T15:03:11.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:10 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/198292946' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/3054125886"}]: dispatch 2024-04-02T15:03:11.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:10 smithi042 ceph-mon[24805]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:11.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:10 smithi042 ceph-mon[29473]: from='client.? ' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6800/3054125886"}]': finished 2024-04-02T15:03:11.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:10 smithi042 ceph-mon[29473]: osdmap e95: 8 total, 8 up, 8 in 2024-04-02T15:03:11.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:10 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/198292946' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/3054125886"}]: dispatch 2024-04-02T15:03:11.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:10 smithi042 ceph-mon[29473]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:11.870 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:11 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/198292946' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/3054125886"}]': finished 2024-04-02T15:03:11.870 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:11 smithi042 ceph-mon[24805]: osdmap e96: 8 total, 8 up, 8 in 2024-04-02T15:03:11.870 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:11 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/198292946' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/3054125886"}]': finished 2024-04-02T15:03:11.870 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:11 smithi042 ceph-mon[29473]: osdmap e96: 8 total, 8 up, 8 in 2024-04-02T15:03:11.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:11 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/198292946' entity='client.iscsi.foo.smithi042.toyfgf' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.42:6801/3054125886"}]': finished 2024-04-02T15:03:11.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:11 smithi169 ceph-mon[25801]: osdmap e96: 8 total, 8 up, 8 in 2024-04-02T15:03:12.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:03:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:03:11] "GET /metrics HTTP/1.1" 200 208099 "" "Prometheus/2.33.4" 2024-04-02T15:03:12.477 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:03:12 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:03:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:03:12.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:12 smithi169 ceph-mon[25801]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T15:03:13.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:12 smithi042 ceph-mon[24805]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T15:03:13.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:12 smithi042 ceph-mon[29473]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T15:03:15.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:15 smithi042 ceph-mon[29473]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-02T15:03:15.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:15 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:15.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:15 smithi042 ceph-mon[24805]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-02T15:03:15.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:15 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:15.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:15 smithi169 ceph-mon[25801]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-02T15:03:15.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:15 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:17.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:17 smithi042 ceph-mon[24805]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T15:03:17.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:17 smithi042 ceph-mon[29473]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T15:03:17.838 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:17 smithi169 ceph-mon[25801]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T15:03:19.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:19 smithi042 ceph-mon[24805]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 576 B/s rd, 0 op/s 2024-04-02T15:03:19.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:19 smithi042 ceph-mon[29473]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 576 B/s rd, 0 op/s 2024-04-02T15:03:19.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:19 smithi169 ceph-mon[25801]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 576 B/s rd, 0 op/s 2024-04-02T15:03:21.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:21 smithi042 ceph-mon[24805]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T15:03:21.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:21 smithi042 ceph-mon[29473]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T15:03:21.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:21 smithi169 ceph-mon[25801]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T15:03:22.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:03:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:03:21] "GET /metrics HTTP/1.1" 200 208086 "" "Prometheus/2.33.4" 2024-04-02T15:03:22.477 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:03:22 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:03:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:03:23.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:23 smithi042 ceph-mon[24805]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-04-02T15:03:23.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:23 smithi042 ceph-mon[29473]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-04-02T15:03:23.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:23 smithi169 ceph-mon[25801]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-04-02T15:03:25.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:25 smithi042 ceph-mon[24805]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:25.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:25 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:25.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:25 smithi042 ceph-mon[29473]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:25.795 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:25 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:25.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:25 smithi169 ceph-mon[25801]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:25.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:25 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:27.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:27 smithi169 ceph-mon[25801]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:27.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:27 smithi042 ceph-mon[24805]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:27.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:27 smithi042 ceph-mon[29473]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:29.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:29 smithi042 ceph-mon[24805]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:29.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:29 smithi042 ceph-mon[29473]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:29.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:29 smithi169 ceph-mon[25801]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:31.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:31 smithi042 ceph-mon[29473]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:31.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:31 smithi042 ceph-mon[24805]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:31.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:31 smithi169 ceph-mon[25801]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:32.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:03:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:03:31] "GET /metrics HTTP/1.1" 200 208086 "" "Prometheus/2.33.4" 2024-04-02T15:03:32.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:03:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:03:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:03:33.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:33 smithi042 ceph-mon[29473]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:33.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:33 smithi042 ceph-mon[24805]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:33.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:33 smithi169 ceph-mon[25801]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:35.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:35 smithi042 ceph-mon[24805]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:35.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:35 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:35.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:35 smithi042 ceph-mon[29473]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:35.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:35 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:35.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:35 smithi169 ceph-mon[25801]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:35.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:35 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:37.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:37 smithi042 ceph-mon[24805]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:37.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:37 smithi042 ceph-mon[29473]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:37.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:37 smithi169 ceph-mon[25801]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:39.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:39 smithi042 ceph-mon[24805]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:39.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:39 smithi042 ceph-mon[29473]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:39.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:39 smithi169 ceph-mon[25801]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:41.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:41 smithi042 ceph-mon[24805]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:41.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:41 smithi042 ceph-mon[29473]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:41.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:41 smithi169 ceph-mon[25801]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:42.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:03:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:03:41] "GET /metrics HTTP/1.1" 200 208040 "" "Prometheus/2.33.4" 2024-04-02T15:03:42.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:03:42 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:03:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:03:43.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:43 smithi042 ceph-mon[24805]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:43.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:43 smithi042 ceph-mon[29473]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:43.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:43 smithi169 ceph-mon[25801]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:45.779 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:45 smithi169 ceph-mon[25801]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:45.779 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:45 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:45.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:45 smithi042 ceph-mon[24805]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:45.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:45 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:45.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:45 smithi042 ceph-mon[29473]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:45.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:45 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:47.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:47 smithi042 ceph-mon[24805]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:47.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:47 smithi042 ceph-mon[29473]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:47.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:47 smithi169 ceph-mon[25801]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:03:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:03:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:03:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:03:48.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:03:48.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:03:48.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:03:48.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:03:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:03:48.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:03:48.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:03:48.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:03:49.774 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:49 smithi042 ceph-mon[24805]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:49.775 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:49 smithi042 ceph-mon[29473]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:49.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:49 smithi169 ceph-mon[25801]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:51.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:51 smithi042 ceph-mon[24805]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:51.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:51 smithi042 ceph-mon[29473]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:51.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:51 smithi169 ceph-mon[25801]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:52.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:03:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:03:51] "GET /metrics HTTP/1.1" 200 207992 "" "Prometheus/2.33.4" 2024-04-02T15:03:52.477 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:03:52 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:03:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:03:53.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:53 smithi042 ceph-mon[29473]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:53.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:53 smithi042 ceph-mon[24805]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:53.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:53 smithi169 ceph-mon[25801]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:55.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:55 smithi042 ceph-mon[24805]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:55.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:55 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:55.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:55 smithi042 ceph-mon[29473]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:55.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:55 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:55.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:55 smithi169 ceph-mon[25801]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:03:55.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:55 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:03:57.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:57 smithi042 ceph-mon[24805]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:57.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:57 smithi042 ceph-mon[29473]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:57.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:57 smithi169 ceph-mon[25801]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:58.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:58 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:03:58.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:58 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:03:58.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:58 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:03:58.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:58 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:03:58.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:58 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:03:58.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:58 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:03:58.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:58 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:03:58.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:58 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:03:58.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:58 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:03:58.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:58 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:03:58.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:58 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:03:58.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:58 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:03:59.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:03:59 smithi042 ceph-mon[24805]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:59.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:03:59 smithi042 ceph-mon[29473]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:03:59.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:03:59 smithi169 ceph-mon[25801]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:01.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:01 smithi042 ceph-mon[24805]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:01.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:01 smithi042 ceph-mon[29473]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:01.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:01 smithi169 ceph-mon[25801]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:02.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:04:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:04:01] "GET /metrics HTTP/1.1" 200 207992 "" "Prometheus/2.33.4" 2024-04-02T15:04:02.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:04:02 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:04:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:04:03.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:03 smithi042 ceph-mon[24805]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:03.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:03 smithi042 ceph-mon[29473]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:03.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:03 smithi169 ceph-mon[25801]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:05.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:05 smithi042 ceph-mon[24805]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:05.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:05 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:05.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:05 smithi042 ceph-mon[29473]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:05.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:05 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:05.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:05 smithi169 ceph-mon[25801]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:05.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:05 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:07.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:07 smithi042 ceph-mon[24805]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:07.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:07 smithi042 ceph-mon[29473]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:07.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:07 smithi169 ceph-mon[25801]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:09.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:09 smithi042 ceph-mon[24805]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:09.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:09 smithi042 ceph-mon[29473]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:09.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:09 smithi169 ceph-mon[25801]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:11.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:11 smithi042 ceph-mon[24805]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:11.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:11 smithi042 ceph-mon[29473]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:11.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:11 smithi169 ceph-mon[25801]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:12.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:04:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:04:11] "GET /metrics HTTP/1.1" 200 208005 "" "Prometheus/2.33.4" 2024-04-02T15:04:12.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:04:12 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:04:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:04:13.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:13 smithi042 ceph-mon[24805]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:13.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:13 smithi042 ceph-mon[29473]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:13.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:13 smithi169 ceph-mon[25801]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:15.779 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:15 smithi169 ceph-mon[25801]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:15.779 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:15 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:15.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:15 smithi042 ceph-mon[24805]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:15.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:15 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:15.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:15 smithi042 ceph-mon[29473]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:15.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:15 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:17.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:17 smithi042 ceph-mon[24805]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:17.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:17 smithi042 ceph-mon[29473]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:17.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:17 smithi169 ceph-mon[25801]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:19.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:19 smithi042 ceph-mon[24805]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:19.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:19 smithi042 ceph-mon[29473]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:19.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:19 smithi169 ceph-mon[25801]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:21.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:21 smithi042 ceph-mon[24805]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:21.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:21 smithi042 ceph-mon[29473]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:21.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:21 smithi169 ceph-mon[25801]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:22.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:04:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:04:21] "GET /metrics HTTP/1.1" 200 208015 "" "Prometheus/2.33.4" 2024-04-02T15:04:22.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:04:22 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:04:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:04:23.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:23 smithi042 ceph-mon[24805]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:23.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:23 smithi042 ceph-mon[29473]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:23.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:23 smithi169 ceph-mon[25801]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:25.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:25 smithi042 ceph-mon[24805]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:25.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:25 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:25.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:25 smithi042 ceph-mon[29473]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:25.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:25 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:25.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:25 smithi169 ceph-mon[25801]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:25.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:25 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:27.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:27 smithi042 ceph-mon[24805]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:27.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:27 smithi042 ceph-mon[29473]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:27.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:27 smithi169 ceph-mon[25801]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:29.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:29 smithi042 ceph-mon[24805]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:29.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:29 smithi042 ceph-mon[29473]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:29.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:29 smithi169 ceph-mon[25801]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:31.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:31 smithi042 ceph-mon[24805]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:31.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:31 smithi042 ceph-mon[29473]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:31.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:31 smithi169 ceph-mon[25801]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:32.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:04:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:04:31] "GET /metrics HTTP/1.1" 200 208015 "" "Prometheus/2.33.4" 2024-04-02T15:04:32.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:04:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:04:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:04:33.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:33 smithi042 ceph-mon[24805]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:33.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:33 smithi042 ceph-mon[29473]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:33.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:33 smithi169 ceph-mon[25801]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:35.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:35 smithi042 ceph-mon[24805]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:35.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:35 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:35.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:35 smithi042 ceph-mon[29473]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:35.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:35 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:35.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:35 smithi169 ceph-mon[25801]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:35.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:35 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:37.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:37 smithi042 ceph-mon[24805]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:37.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:37 smithi042 ceph-mon[29473]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:37.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:37 smithi169 ceph-mon[25801]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:39.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:39 smithi042 ceph-mon[24805]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:39.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:39 smithi042 ceph-mon[29473]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:39.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:39 smithi169 ceph-mon[25801]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:41.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:41 smithi042 ceph-mon[24805]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:41.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:41 smithi042 ceph-mon[29473]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:41.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:41 smithi169 ceph-mon[25801]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:42.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:04:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:04:41] "GET /metrics HTTP/1.1" 200 208013 "" "Prometheus/2.33.4" 2024-04-02T15:04:42.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:04:42 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:04:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:04:43.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:43 smithi042 ceph-mon[24805]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:43.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:43 smithi042 ceph-mon[29473]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:43.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:43 smithi169 ceph-mon[25801]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:45.779 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:45 smithi169 ceph-mon[25801]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:45.779 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:45 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:45.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:45 smithi042 ceph-mon[29473]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:45.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:45 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:45.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:45 smithi042 ceph-mon[24805]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:45.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:45 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:47.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:47 smithi042 ceph-mon[24805]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:47.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:47 smithi042 ceph-mon[29473]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:47.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:47 smithi169 ceph-mon[25801]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:04:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:04:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:04:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:04:48.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:04:48.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:04:48.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:04:48.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:04:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:04:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:04:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:04:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:04:49.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:49 smithi042 ceph-mon[24805]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T15:04:49.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:49 smithi042 ceph-mon[29473]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T15:04:49.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:49 smithi169 ceph-mon[25801]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T15:04:51.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:51 smithi169 ceph-mon[25801]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:51.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:51 smithi042 ceph-mon[24805]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:51.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:51 smithi042 ceph-mon[29473]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:52.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:04:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:04:51] "GET /metrics HTTP/1.1" 200 208005 "" "Prometheus/2.33.4" 2024-04-02T15:04:52.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:04:52 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:04:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:04:53.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:53 smithi042 ceph-mon[24805]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T15:04:53.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:53 smithi042 ceph-mon[29473]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T15:04:53.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:53 smithi169 ceph-mon[25801]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T15:04:55.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:55 smithi042 ceph-mon[24805]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:55.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:55 smithi042 ceph-mon[29473]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:55.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:55 smithi169 ceph-mon[25801]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:04:56.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:56 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:56.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:56 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:56.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:56 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:04:57.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:57 smithi042 ceph-mon[24805]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T15:04:57.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:57 smithi042 ceph-mon[29473]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T15:04:57.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:57 smithi169 ceph-mon[25801]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T15:04:58.771 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:58 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:04:58.772 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:58 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:04:58.772 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:58 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:04:58.775 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:58 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:04:58.775 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:58 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:04:58.776 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:58 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:04:58.776 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:58 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:04:58.776 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:58 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:04:58.776 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:58 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:04:59.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:59 smithi042 ceph-mon[29473]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:59.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:04:59 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:04:59.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:59 smithi042 ceph-mon[24805]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:59.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:04:59 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:04:59.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:59 smithi169 ceph-mon[25801]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:04:59.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:04:59 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:05:01.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:01 smithi042 ceph-mon[24805]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:01.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:01 smithi042 ceph-mon[29473]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:01.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:01 smithi169 ceph-mon[25801]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:02.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:05:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:05:01] "GET /metrics HTTP/1.1" 200 208005 "" "Prometheus/2.33.4" 2024-04-02T15:05:02.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:05:02 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:05:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:05:03.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:03 smithi042 ceph-mon[24805]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:03.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:03 smithi042 ceph-mon[29473]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:03.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:03 smithi169 ceph-mon[25801]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:05.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:05 smithi042 ceph-mon[24805]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:05.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:05 smithi042 ceph-mon[29473]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:05.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:05 smithi169 ceph-mon[25801]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:06.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:06 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:06.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:06 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:06.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:06 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:07.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:07 smithi042 ceph-mon[24805]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:07.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:07 smithi042 ceph-mon[29473]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:07.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:07 smithi169 ceph-mon[25801]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:08.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:08 smithi042 ceph-mon[24805]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:08.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:08 smithi042 ceph-mon[29473]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:08.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:08 smithi169 ceph-mon[25801]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:11.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:11 smithi042 ceph-mon[24805]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:11.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:11 smithi042 ceph-mon[29473]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:11.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:11 smithi169 ceph-mon[25801]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:12.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:05:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:05:11] "GET /metrics HTTP/1.1" 200 208003 "" "Prometheus/2.33.4" 2024-04-02T15:05:12.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:05:12 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:05:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:05:13.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:13 smithi042 ceph-mon[24805]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:13.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:13 smithi042 ceph-mon[29473]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:13.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:13 smithi169 ceph-mon[25801]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:15.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:15 smithi042 ceph-mon[24805]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:15.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:15 smithi042 ceph-mon[29473]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:15.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:15 smithi169 ceph-mon[25801]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:16.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:16 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:16.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:16 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:16.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:16 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:17.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:17 smithi042 ceph-mon[24805]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:17.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:17 smithi042 ceph-mon[29473]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:17.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:17 smithi169 ceph-mon[25801]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:18.897 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:18 smithi042 ceph-mon[24805]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:18.898 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:18 smithi042 ceph-mon[29473]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:18.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:18 smithi169 ceph-mon[25801]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:21.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:21 smithi042 ceph-mon[24805]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:21.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:21 smithi042 ceph-mon[29473]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:21.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:21 smithi169 ceph-mon[25801]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:22.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:05:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:05:21] "GET /metrics HTTP/1.1" 200 207940 "" "Prometheus/2.33.4" 2024-04-02T15:05:22.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:05:22 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:05:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:05:23.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:23 smithi042 ceph-mon[24805]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:23.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:23 smithi042 ceph-mon[29473]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:23.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:23 smithi169 ceph-mon[25801]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:25.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:25 smithi042 ceph-mon[24805]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:25.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:25 smithi042 ceph-mon[29473]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:25.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:25 smithi169 ceph-mon[25801]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:26.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:26 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:27.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:26 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:27.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:26 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:27.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:27 smithi169 ceph-mon[25801]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:28.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:27 smithi042 ceph-mon[24805]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:28.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:27 smithi042 ceph-mon[29473]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:28.794 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:28 smithi042 ceph-mon[24805]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:28.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:28 smithi042 ceph-mon[29473]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:28.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:28 smithi169 ceph-mon[25801]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:31.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:31 smithi042 ceph-mon[24805]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:31.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:31 smithi042 ceph-mon[29473]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:31.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:31 smithi169 ceph-mon[25801]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:32.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:05:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:05:31] "GET /metrics HTTP/1.1" 200 207940 "" "Prometheus/2.33.4" 2024-04-02T15:05:32.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:05:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:05:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:05:33.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:33 smithi042 ceph-mon[24805]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:33.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:33 smithi042 ceph-mon[29473]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:33.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:33 smithi169 ceph-mon[25801]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:35.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:35 smithi042 ceph-mon[24805]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:35.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:35 smithi042 ceph-mon[29473]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:35.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:35 smithi169 ceph-mon[25801]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:36.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:36 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:37.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:36 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:37.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:36 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:37.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:37 smithi169 ceph-mon[25801]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:38.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:37 smithi042 ceph-mon[24805]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:38.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:37 smithi042 ceph-mon[29473]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:38.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:38 smithi169 ceph-mon[25801]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:39.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:38 smithi042 ceph-mon[24805]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:39.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:38 smithi042 ceph-mon[29473]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:41.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:41 smithi042 ceph-mon[24805]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:41.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:41 smithi042 ceph-mon[29473]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:41.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:41 smithi169 ceph-mon[25801]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:42.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:05:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:05:41] "GET /metrics HTTP/1.1" 200 207957 "" "Prometheus/2.33.4" 2024-04-02T15:05:42.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:05:42 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:05:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:05:43.133 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-02T15:05:43.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:43 smithi042 ceph-mon[24805]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:43.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:43 smithi042 ceph-mon[29473]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:43.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:43 smithi169 ceph-mon[25801]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:44.732 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:44 smithi042 ceph-mon[24805]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:44.732 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:44 smithi042 ceph-mon[29473]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:44.980 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:44 smithi169 ceph-mon[25801]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:45.933 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-02T15:05:46.499 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:46 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:46.500 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:46 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:46.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:46 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:47.377 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:47 smithi042 ceph-mon[29473]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:47.377 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:47 smithi042 ceph-mon[24805]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:47.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:47 smithi169 ceph-mon[25801]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:48.487 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-02T15:05:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:48 smithi042 ceph-mon[24805]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:05:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:05:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:05:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:05:48.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:48 smithi042 ceph-mon[29473]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:48.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:05:48.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:05:48.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:05:48.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:05:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:48 smithi169 ceph-mon[25801]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:05:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:05:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:05:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:05:51.244 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch ps' 2024-04-02T15:05:51.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:51 smithi042 ceph-mon[24805]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:51.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:51 smithi042 ceph-mon[29473]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:51.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:51 smithi169 ceph-mon[25801]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:52.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:05:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:05:51] "GET /metrics HTTP/1.1" 200 207951 "" "Prometheus/2.33.4" 2024-04-02T15:05:52.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:05:52 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:05:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:05:53.272 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T15:05:53.273 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (3m) 2m ago 3m 15.5M - ba2b418f427c 64e49f76da9a 2024-04-02T15:05:53.273 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi169 *:3000 running (3m) 2m ago 3m 46.9M - 8.3.5 dad864ee21e9 acd65a3d3b6a 2024-04-02T15:05:53.273 INFO:teuthology.orchestra.run.smithi042.stdout:iscsi.foo.smithi042.toyfgf smithi042 running (2m) 2m ago 2m 66.7M - 3.5 e1d6a67b021e b024dc64805f 2024-04-02T15:05:53.273 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi169 *:8443 running (8m) 2m ago 8m 411M - 17.2.0 e1d6a67b021e 794578fdbdb9 2024-04-02T15:05:53.273 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:9283 running (10m) 2m ago 10m 454M - 17.2.0 e1d6a67b021e 92a15ec7fb5f 2024-04-02T15:05:53.273 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (10m) 2m ago 10m 46.7M 2048M 17.2.0 e1d6a67b021e d1df5488c219 2024-04-02T15:05:53.273 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi169 running (8m) 2m ago 8m 42.1M 2048M 17.2.0 e1d6a67b021e 1cc495c7e481 2024-04-02T15:05:53.273 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (8m) 2m ago 8m 42.5M 2048M 17.2.0 e1d6a67b021e dbd3308e4000 2024-04-02T15:05:53.273 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (4m) 2m ago 4m 18.7M - 1dbe0e931976 d0a5ca877a2a 2024-04-02T15:05:53.273 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi169 *:9100 running (4m) 2m ago 4m 13.9M - 1dbe0e931976 dbb30e649ccf 2024-04-02T15:05:53.273 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (8m) 2m ago 8m 49.6M 4017M 17.2.0 e1d6a67b021e 590bee772da0 2024-04-02T15:05:53.273 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (7m) 2m ago 7m 48.8M 4017M 17.2.0 e1d6a67b021e 32f3310fd9f9 2024-04-02T15:05:53.273 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (7m) 2m ago 7m 44.7M 4017M 17.2.0 e1d6a67b021e f2d6ce6b9bfa 2024-04-02T15:05:53.274 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (6m) 2m ago 6m 45.7M 4017M 17.2.0 e1d6a67b021e c5d80c30db4c 2024-04-02T15:05:53.274 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi169 running (6m) 2m ago 6m 46.6M 4272M 17.2.0 e1d6a67b021e efc569199c21 2024-04-02T15:05:53.274 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi169 running (6m) 2m ago 6m 43.7M 4272M 17.2.0 e1d6a67b021e d543be94df6d 2024-04-02T15:05:53.274 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi169 running (5m) 2m ago 5m 42.6M 4272M 17.2.0 e1d6a67b021e 34c8a641d867 2024-04-02T15:05:53.274 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi169 running (5m) 2m ago 5m 46.1M 4272M 17.2.0 e1d6a67b021e 817b439db33c 2024-04-02T15:05:53.274 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi169 *:9095 running (3m) 2m ago 4m 43.1M - 514e6a882f6e 80b1a5d12473 2024-04-02T15:05:53.274 INFO:teuthology.orchestra.run.smithi042.stdout:rgw.foo.smithi042.plghvb smithi042 *:8000 running (3m) 2m ago 3m 70.4M - 17.2.0 e1d6a67b021e a220ab61cedb 2024-04-02T15:05:53.274 INFO:teuthology.orchestra.run.smithi042.stdout:rgw.foo.smithi169.pvuzch smithi169 *:8000 running (3m) 2m ago 3m 68.7M - 17.2.0 e1d6a67b021e 7e7d6aa0d138 2024-04-02T15:05:53.976 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:53 smithi042 ceph-mon[24805]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:53.976 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:53 smithi042 ceph-mon[29473]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:53.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:53 smithi169 ceph-mon[25801]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:54.002 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph versions' 2024-04-02T15:05:54.570 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:54 smithi042 ceph-mon[24805]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:05:54.571 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:54 smithi042 ceph-mon[24805]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:54.571 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:54 smithi042 ceph-mon[29473]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:05:54.571 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:54 smithi042 ceph-mon[29473]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:54.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:54 smithi169 ceph-mon[25801]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:05:54.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:54 smithi169 ceph-mon[25801]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:05:56.029 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-04-02T15:05:56.030 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-04-02T15:05:56.030 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-02T15:05:56.030 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:05:56.030 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-04-02T15:05:56.030 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T15:05:56.030 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:05:56.030 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-04-02T15:05:56.030 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T15:05:56.030 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:05:56.030 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": {}, 2024-04-02T15:05:56.030 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw": { 2024-04-02T15:05:56.030 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T15:05:56.030 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:05:56.031 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-04-02T15:05:56.031 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-04-02T15:05:56.031 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-04-02T15:05:56.031 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-04-02T15:05:56.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:56 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:56.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:56 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/535320417' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:05:56.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:56 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:56.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:56 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/535320417' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:05:56.671 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph -s' 2024-04-02T15:05:56.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:56 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:05:56.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:56 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/535320417' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:05:57.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:57 smithi042 ceph-mon[29473]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:57.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:57 smithi042 ceph-mon[24805]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:57.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:57 smithi169 ceph-mon[25801]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:58.695 INFO:teuthology.orchestra.run.smithi042.stdout: cluster: 2024-04-02T15:05:58.695 INFO:teuthology.orchestra.run.smithi042.stdout: id: fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T15:05:58.695 INFO:teuthology.orchestra.run.smithi042.stdout: health: HEALTH_OK 2024-04-02T15:05:58.695 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:05:58.696 INFO:teuthology.orchestra.run.smithi042.stdout: services: 2024-04-02T15:05:58.696 INFO:teuthology.orchestra.run.smithi042.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-04-02T15:05:58.696 INFO:teuthology.orchestra.run.smithi042.stdout: mgr: y(active, since 5m), standbys: x 2024-04-02T15:05:58.696 INFO:teuthology.orchestra.run.smithi042.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-04-02T15:05:58.696 INFO:teuthology.orchestra.run.smithi042.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-02T15:05:58.696 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:05:58.696 INFO:teuthology.orchestra.run.smithi042.stdout: data: 2024-04-02T15:05:58.696 INFO:teuthology.orchestra.run.smithi042.stdout: pools: 6 pools, 161 pgs 2024-04-02T15:05:58.696 INFO:teuthology.orchestra.run.smithi042.stdout: objects: 209 objects, 585 KiB 2024-04-02T15:05:58.696 INFO:teuthology.orchestra.run.smithi042.stdout: usage: 68 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:05:58.696 INFO:teuthology.orchestra.run.smithi042.stdout: pgs: 161 active+clean 2024-04-02T15:05:58.696 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:05:58.696 INFO:teuthology.orchestra.run.smithi042.stdout: io: 2024-04-02T15:05:58.697 INFO:teuthology.orchestra.run.smithi042.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-02T15:05:58.697 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:05:59.471 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch ls' 2024-04-02T15:05:59.785 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:59 smithi042 ceph-mon[29473]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:59.785 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:59 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/267876071' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T15:05:59.785 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:59 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:05:59.785 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:59 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:05:59.785 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:05:59 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:05:59.785 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:59 smithi042 ceph-mon[24805]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:59.785 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:59 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/267876071' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T15:05:59.785 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:59 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:05:59.785 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:59 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:05:59.785 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:05:59 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:05:59.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:59 smithi169 ceph-mon[25801]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:05:59.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:59 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/267876071' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T15:05:59.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:59 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:05:59.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:59 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:05:59.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:05:59 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:06:00.799 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:00 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:00.799 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:00 smithi042 ceph-mon[24805]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:00.799 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:00 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:00.800 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:00 smithi042 ceph-mon[29473]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:00.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:00 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:00.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:00 smithi169 ceph-mon[25801]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:01.595 INFO:teuthology.orchestra.run.smithi042.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-02T15:06:01.595 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi042=a;count:1 2024-04-02T15:06:01.595 INFO:teuthology.orchestra.run.smithi042.stdout:grafana ?:3000 1/1 3m ago 5m smithi169=a;count:1 2024-04-02T15:06:01.595 INFO:teuthology.orchestra.run.smithi042.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-04-02T15:06:01.595 INFO:teuthology.orchestra.run.smithi042.stdout:mgr 2/2 3m ago 8m smithi042=y;smithi169=x;count:2 2024-04-02T15:06:01.595 INFO:teuthology.orchestra.run.smithi042.stdout:mon 3/3 3m ago 9m smithi042:172.21.15.42=a;smithi042:[v2:172.21.15.42:3301,v1:172.21.15.42:6790]=c;smithi169:172.21.15.169=b;count:3 2024-04-02T15:06:01.595 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi042=a;smithi169=b;count:2 2024-04-02T15:06:01.595 INFO:teuthology.orchestra.run.smithi042.stdout:osd 8 3m ago - 2024-04-02T15:06:01.595 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus ?:9095 1/1 3m ago 5m smithi169=a;count:1 2024-04-02T15:06:01.595 INFO:teuthology.orchestra.run.smithi042.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-04-02T15:06:02.241 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:06:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:06:01] "GET /metrics HTTP/1.1" 200 207951 "" "Prometheus/2.33.4" 2024-04-02T15:06:02.267 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-02T15:06:02.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:06:02 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:06:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:06:03.838 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:03 smithi042 ceph-mon[24805]: from='client.14985 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:06:03.838 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:03 smithi042 ceph-mon[24805]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:03.838 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:03 smithi042 ceph-mon[29473]: from='client.14985 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:06:03.838 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:03 smithi042 ceph-mon[29473]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:03.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:03 smithi169 ceph-mon[25801]: from='client.14985 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:06:03.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:03 smithi169 ceph-mon[25801]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:04.752 INFO:teuthology.orchestra.run.smithi042.stdout:Scheduled to redeploy mgr.x on host 'smithi169' 2024-04-02T15:06:04.815 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:04 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/3095927077' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:06:04.816 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:04 smithi042 ceph-mon[24805]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:04.816 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:04 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/3095927077' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:06:04.816 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:04 smithi042 ceph-mon[29473]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:04.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:04 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/3095927077' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:06:04.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:04 smithi169 ceph-mon[25801]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:05.521 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch ps --refresh' 2024-04-02T15:06:06.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:05 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:06.483 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:05 smithi042 ceph-mon[29473]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:3f3f3ed992429b85be729743ad3b39d52dd90744", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:06:06.483 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:05 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:06.483 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:05 smithi042 ceph-mon[29473]: Schedule redeploy daemon mgr.x 2024-04-02T15:06:06.483 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:05 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:06:06.484 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:05 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:06.484 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:05 smithi042 ceph-mon[24805]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:3f3f3ed992429b85be729743ad3b39d52dd90744", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:06:06.484 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:05 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:06.484 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:05 smithi042 ceph-mon[24805]: Schedule redeploy daemon mgr.x 2024-04-02T15:06:06.484 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:05 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:06:06.484 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:05 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:06.484 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:05 smithi169 ceph-mon[25801]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:3f3f3ed992429b85be729743ad3b39d52dd90744", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:06:06.485 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:05 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:06.485 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:05 smithi169 ceph-mon[25801]: Schedule redeploy daemon mgr.x 2024-04-02T15:06:06.485 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:05 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:06:07.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:06 smithi042 ceph-mon[29473]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:07.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:06 smithi042 ceph-mon[24805]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:07.226 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:06 smithi169 ceph-mon[25801]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:07.897 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T15:06:07.897 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (3m) 3m ago 4m 15.5M - ba2b418f427c 64e49f76da9a 2024-04-02T15:06:07.897 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi169 *:3000 running (3m) 3m ago 3m 46.9M - 8.3.5 dad864ee21e9 acd65a3d3b6a 2024-04-02T15:06:07.897 INFO:teuthology.orchestra.run.smithi042.stdout:iscsi.foo.smithi042.toyfgf smithi042 running (3m) 3m ago 3m 66.7M - 3.5 e1d6a67b021e b024dc64805f 2024-04-02T15:06:07.897 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi169 *:8443 running (8m) 3m ago 8m 411M - 17.2.0 e1d6a67b021e 794578fdbdb9 2024-04-02T15:06:07.897 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:9283 running (10m) 3m ago 10m 454M - 17.2.0 e1d6a67b021e 92a15ec7fb5f 2024-04-02T15:06:07.897 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (10m) 3m ago 10m 46.7M 2048M 17.2.0 e1d6a67b021e d1df5488c219 2024-04-02T15:06:07.897 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi169 running (8m) 3m ago 8m 42.1M 2048M 17.2.0 e1d6a67b021e 1cc495c7e481 2024-04-02T15:06:07.897 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (9m) 3m ago 9m 42.5M 2048M 17.2.0 e1d6a67b021e dbd3308e4000 2024-04-02T15:06:07.897 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (4m) 3m ago 4m 18.7M - 1dbe0e931976 d0a5ca877a2a 2024-04-02T15:06:07.897 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi169 *:9100 running (4m) 3m ago 4m 13.9M - 1dbe0e931976 dbb30e649ccf 2024-04-02T15:06:07.898 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (8m) 3m ago 8m 49.6M 4017M 17.2.0 e1d6a67b021e 590bee772da0 2024-04-02T15:06:07.898 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (8m) 3m ago 8m 48.8M 4017M 17.2.0 e1d6a67b021e 32f3310fd9f9 2024-04-02T15:06:07.898 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (7m) 3m ago 7m 44.7M 4017M 17.2.0 e1d6a67b021e f2d6ce6b9bfa 2024-04-02T15:06:07.898 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (7m) 3m ago 7m 45.7M 4017M 17.2.0 e1d6a67b021e c5d80c30db4c 2024-04-02T15:06:07.898 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi169 running (6m) 3m ago 6m 46.6M 4272M 17.2.0 e1d6a67b021e efc569199c21 2024-04-02T15:06:07.898 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi169 running (6m) 3m ago 6m 43.7M 4272M 17.2.0 e1d6a67b021e d543be94df6d 2024-04-02T15:06:07.898 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi169 running (6m) 3m ago 6m 42.6M 4272M 17.2.0 e1d6a67b021e 34c8a641d867 2024-04-02T15:06:07.898 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi169 running (5m) 3m ago 5m 46.1M 4272M 17.2.0 e1d6a67b021e 817b439db33c 2024-04-02T15:06:07.898 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi169 *:9095 running (3m) 3m ago 5m 43.1M - 514e6a882f6e 80b1a5d12473 2024-04-02T15:06:07.898 INFO:teuthology.orchestra.run.smithi042.stdout:rgw.foo.smithi042.plghvb smithi042 *:8000 running (3m) 3m ago 3m 70.4M - 17.2.0 e1d6a67b021e a220ab61cedb 2024-04-02T15:06:07.898 INFO:teuthology.orchestra.run.smithi042.stdout:rgw.foo.smithi169.pvuzch smithi169 *:8000 running (3m) 3m ago 3m 68.7M - 17.2.0 e1d6a67b021e 7e7d6aa0d138 2024-04-02T15:06:08.645 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'sleep 180' 2024-04-02T15:06:08.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:08 smithi169 ceph-mon[25801]: from='client.24928 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:06:08.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:08 smithi169 ceph-mon[25801]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:08.983 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:08 smithi042 ceph-mon[29473]: from='client.24928 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:06:08.983 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:08 smithi042 ceph-mon[29473]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:08.983 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:08 smithi042 ceph-mon[24805]: from='client.24928 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:06:08.983 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:08 smithi042 ceph-mon[24805]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:11.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:11 smithi042 ceph-mon[24805]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:11.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:11 smithi042 ceph-mon[29473]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:11.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:11 smithi169 ceph-mon[25801]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:12.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:06:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:06:11] "GET /metrics HTTP/1.1" 200 207936 "" "Prometheus/2.33.4" 2024-04-02T15:06:12.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:06:12 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:06:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:06:12.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:12 smithi169 ceph-mon[25801]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:13.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:12 smithi042 ceph-mon[29473]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:13.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:12 smithi042 ceph-mon[24805]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:14.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:14 smithi169 ceph-mon[25801]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:15.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:14 smithi042 ceph-mon[24805]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:15.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:14 smithi042 ceph-mon[29473]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:16.541 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:16 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:16.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:16 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:16.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:16 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:17.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:17 smithi169 ceph-mon[25801]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:17.541 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:17 smithi042 ceph-mon[24805]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:17.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:17 smithi042 ceph-mon[29473]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:18.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:18 smithi169 ceph-mon[25801]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:19.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:18 smithi042 ceph-mon[24805]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:19.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:18 smithi042 ceph-mon[29473]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:21.541 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:21 smithi042 ceph-mon[24805]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:21.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:21 smithi042 ceph-mon[29473]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:21.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:21 smithi169 ceph-mon[25801]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:22.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:06:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:06:21] "GET /metrics HTTP/1.1" 200 207944 "" "Prometheus/2.33.4" 2024-04-02T15:06:22.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:06:22 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:06:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:06:22.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:22 smithi169 ceph-mon[25801]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:23.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:22 smithi042 ceph-mon[24805]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:23.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:22 smithi042 ceph-mon[29473]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:24.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:24 smithi169 ceph-mon[25801]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:25.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:24 smithi042 ceph-mon[24805]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:25.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:24 smithi042 ceph-mon[29473]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:26.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:26 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:26.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:26 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:26.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:26 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:27.541 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:27 smithi042 ceph-mon[24805]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:27.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:27 smithi042 ceph-mon[29473]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:27.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:27 smithi169 ceph-mon[25801]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:28.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:28 smithi042 ceph-mon[24805]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:28.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:28 smithi042 ceph-mon[29473]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:28.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:28 smithi169 ceph-mon[25801]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:31.541 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:31 smithi042 ceph-mon[24805]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:31.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:31 smithi042 ceph-mon[29473]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:31.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:31 smithi169 ceph-mon[25801]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:32.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:06:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:06:31] "GET /metrics HTTP/1.1" 200 207944 "" "Prometheus/2.33.4" 2024-04-02T15:06:32.477 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:06:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:06:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:06:32.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:32 smithi169 ceph-mon[25801]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:33.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:32 smithi042 ceph-mon[24805]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:33.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:32 smithi042 ceph-mon[29473]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:34.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:34 smithi169 ceph-mon[25801]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:35.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:34 smithi042 ceph-mon[24805]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:35.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:34 smithi042 ceph-mon[29473]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:36.541 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:36 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:36.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:36 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:36.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:36 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:37.541 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:37 smithi042 ceph-mon[24805]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:37.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:37 smithi042 ceph-mon[29473]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:37.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:37 smithi169 ceph-mon[25801]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:38.855 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:38 smithi042 ceph-mon[24805]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:38.855 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:38 smithi042 ceph-mon[29473]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:38.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:38 smithi169 ceph-mon[25801]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:41.291 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:41 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:41.292 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:41 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:06:41.292 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:41 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:06:41.292 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:41 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:41.292 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:41 smithi042 ceph-mon[24805]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:41.292 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:41 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:41.292 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:41 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:06:41.292 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:41 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:06:41.293 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:41 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:41.293 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:41 smithi042 ceph-mon[29473]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:41.360 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:41 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:41.360 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:41 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:06:41.360 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:41 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:06:41.360 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:41 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:41.360 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:41 smithi169 ceph-mon[25801]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:42.036 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:06:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:06:41] "GET /metrics HTTP/1.1" 200 207947 "" "Prometheus/2.33.4" 2024-04-02T15:06:42.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:06:42 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:06:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:06:42.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:42 smithi042 ceph-mon[24805]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:42.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:42 smithi042 ceph-mon[29473]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:42.867 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:42 smithi169 ceph-mon[25801]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:44.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:44 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:44.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:44 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:44.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:44 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:44.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:44 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T15:06:44.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:44 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T15:06:44.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:44 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T15:06:44.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:44 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:06:44.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:44 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:44.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:44 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:44.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:44 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:44.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:44 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T15:06:44.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:44 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T15:06:44.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:44 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T15:06:44.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:44 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:06:44.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:44 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:44.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:44 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:44.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:44 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:06:44.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:44 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T15:06:44.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:44 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T15:06:44.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:44 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T15:06:44.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:44 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:06:45.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:45 smithi169 ceph-mon[25801]: Deploying daemon mgr.x on smithi169 2024-04-02T15:06:45.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:45 smithi169 ceph-mon[25801]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:45.541 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:45 smithi042 ceph-mon[29473]: Deploying daemon mgr.x on smithi169 2024-04-02T15:06:45.541 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:45 smithi042 ceph-mon[29473]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:45.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:45 smithi042 ceph-mon[24805]: Deploying daemon mgr.x on smithi169 2024-04-02T15:06:45.542 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:45 smithi042 ceph-mon[24805]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:46.541 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:46 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:46.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:46 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:46.686 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:46 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:47.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:47 smithi169 ceph-mon[25801]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:47.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:47 smithi042 ceph-mon[24805]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:47.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:47 smithi042 ceph-mon[29473]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:48 smithi042 ceph-mon[24805]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:06:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:06:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:06:48.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:06:48.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:48 smithi042 ceph-mon[29473]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:48.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:06:48.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:06:48.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:06:48.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:06:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:48 smithi169 ceph-mon[25801]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:06:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:06:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:06:48.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:06:51.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:51 smithi169 ceph-mon[25801]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:51.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:51 smithi042 ceph-mon[24805]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:51.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:51 smithi042 ceph-mon[29473]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:52.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:06:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:06:51] "GET /metrics HTTP/1.1" 200 207949 "" "Prometheus/2.33.4" 2024-04-02T15:06:52.475 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:06:52 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:06:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:06:52.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:52 smithi169 ceph-mon[25801]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:53.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:52 smithi042 ceph-mon[29473]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:53.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:52 smithi042 ceph-mon[24805]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:53.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:06:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:06:53.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.169 because it doesn't contain any IP SANs" 2024-04-02T15:06:53.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:06:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:06:53.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:06:54.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:54 smithi169 ceph-mon[25801]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:55.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:54 smithi042 ceph-mon[24805]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:55.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:54 smithi042 ceph-mon[29473]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:06:56.541 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:56 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:56.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:56 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:56.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:56 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:06:57.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:57 smithi169 ceph-mon[25801]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:57.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:57 smithi042 ceph-mon[24805]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:57.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:57 smithi042 ceph-mon[29473]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:58.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:06:58 smithi169 ceph-mon[25801]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:59.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:06:58 smithi042 ceph-mon[24805]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:06:59.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:06:58 smithi042 ceph-mon[29473]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:01.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:01 smithi169 ceph-mon[25801]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:01.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:01 smithi042 ceph-mon[24805]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:01.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:01 smithi042 ceph-mon[29473]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:02.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:07:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:07:01] "GET /metrics HTTP/1.1" 200 207949 "" "Prometheus/2.33.4" 2024-04-02T15:07:02.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:02 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:07:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:07:02.982 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:02 smithi169 ceph-mon[25801]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:03.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:02 smithi042 ceph-mon[24805]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:03.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:02 smithi042 ceph-mon[29473]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:03.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:07:03.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.169 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:07:03.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:07:03.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.169 because it doesn't contain any IP SANs" 2024-04-02T15:07:03.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:07:03.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:07:04.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:04 smithi169 ceph-mon[25801]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:05.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:04 smithi042 ceph-mon[24805]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:05.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:04 smithi042 ceph-mon[29473]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:06.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:06 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:06.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:06 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:06.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:06 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:07.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:07 smithi169 ceph-mon[25801]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:07.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:07 smithi042 ceph-mon[24805]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:07.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:07 smithi042 ceph-mon[29473]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:08.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:08 smithi169 ceph-mon[25801]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:09.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:08 smithi042 ceph-mon[29473]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:09.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:08 smithi042 ceph-mon[24805]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:11.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:11 smithi169 ceph-mon[25801]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:11.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:11 smithi042 ceph-mon[24805]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:11.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:11 smithi042 ceph-mon[29473]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:12.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:07:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:07:11] "GET /metrics HTTP/1.1" 200 207968 "" "Prometheus/2.33.4" 2024-04-02T15:07:12.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:12 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[26981]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:07:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:07:12.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:12 smithi169 ceph-mon[25801]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:13.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:12 smithi042 ceph-mon[24805]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:13.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:12 smithi042 ceph-mon[29473]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:13.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:07:13.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.169 because it doesn't contain any IP SANs" 2024-04-02T15:07:13.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:07:13.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.169 because it doesn't contain any IP SANs" 2024-04-02T15:07:13.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:07:13.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:07:14.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:14 smithi169 ceph-mon[25801]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:15.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:14 smithi042 ceph-mon[24805]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:15.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:14 smithi042 ceph-mon[29473]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:16.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:16 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:16.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:16 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:16.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:16 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:17.158 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:17 smithi169 systemd[1]: Stopping Ceph mgr.x for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:07:17.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:17 smithi169 ceph-mon[25801]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:17.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:17 smithi169 bash[45864]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr.x" found: no such container 2024-04-02T15:07:17.476 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:17 smithi169 podman[45878]: 2024-04-02 15:07:17.304757403 +0000 UTC m=+0.124679996 container died 794578fdbdb93f619994fc2d7cffce882fad4f6b6e5555cdb5140388171f4ef2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, architecture=x86_64, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , version=8, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, io.openshift.expose-services=, name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T15:07:17.541 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:17 smithi042 ceph-mon[24805]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:17.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:17 smithi042 ceph-mon[29473]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:17.542 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:17 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:07:17.302Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=5 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": read tcp 172.21.15.42:42838->172.21.15.169:8443: read: connection reset by peer" 2024-04-02T15:07:17.881 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:17 smithi169 podman[45878]: 2024-04-02 15:07:17.701073566 +0000 UTC m=+0.520996174 container cleanup 794578fdbdb93f619994fc2d7cffce882fad4f6b6e5555cdb5140388171f4ef2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x, name=centos-stream, version=8, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , ceph=True, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, architecture=x86_64, release=754, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-02T15:07:17.881 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:17 smithi169 bash[45878]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x 2024-04-02T15:07:18.226 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:17 smithi169 podman[45892]: 2024-04-02 15:07:17.894090336 +0000 UTC m=+0.586192133 container remove 794578fdbdb93f619994fc2d7cffce882fad4f6b6e5555cdb5140388171f4ef2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_BRANCH=HEAD, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, vcs-type=git, io.buildah.version=1.19.8) 2024-04-02T15:07:18.226 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:17 smithi169 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-02T15:07:18.226 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:17 smithi169 bash[45904]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr.x" found: no such container 2024-04-02T15:07:18.226 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:17 smithi169 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mgr.x.service: Failed with result 'exit-code'. 2024-04-02T15:07:18.226 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:17 smithi169 systemd[1]: Stopped Ceph mgr.x for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T15:07:18.226 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:17 smithi169 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mgr.x.service: Consumed 17.120s CPU time. 2024-04-02T15:07:18.226 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 systemd[1]: Starting Ceph mgr.x for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:07:18.585 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 podman[46008]: 2024-04-02 15:07:18.240931231 +0000 UTC m=+0.018293383 image pull 7661fe7521b944a40c63d099d9c8c6454b2d4a3af89d0e01e094f668bed96783 quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac 2024-04-02T15:07:18.586 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 podman[46008]: 2024-04-02 15:07:18.470367681 +0000 UTC m=+0.247729847 container create 4219b532beb653631b6eb5f1cf16a3c843222d10c8f5f06c7432cc6855a23f19 (image=quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-04-02-0703-3f3f3ed, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True) 2024-04-02T15:07:18.849 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-mon[25801]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:18.850 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 podman[46008]: 2024-04-02 15:07:18.714807842 +0000 UTC m=+0.492169961 container init 4219b532beb653631b6eb5f1cf16a3c843222d10c8f5f06c7432cc6855a23f19 (image=quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-04-02-0703-3f3f3ed, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-02T15:07:18.850 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 podman[46008]: 2024-04-02 15:07:18.717674419 +0000 UTC m=+0.495036548 container start 4219b532beb653631b6eb5f1cf16a3c843222d10c8f5f06c7432cc6855a23f19 (image=quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-04-02-0703-3f3f3ed, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, ceph=True) 2024-04-02T15:07:18.850 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-mgr[46027]: -- 172.21.15.169:0/743533280 <== mon.2 v2:172.21.15.169:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5650a6f4c5a0 con 0x5650a6f20800 2024-04-02T15:07:18.850 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.776+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:18.850 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.776+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:18.850 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.850 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.780+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'restful' 2024-04-02T15:07:18.850 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.780+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'restful': (22) Invalid argument 2024-04-02T15:07:18.850 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.797+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:18.850 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 bash[46008]: 4219b532beb653631b6eb5f1cf16a3c843222d10c8f5f06c7432cc6855a23f19 2024-04-02T15:07:18.850 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 systemd[1]: Started Ceph mgr.x for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T15:07:18.850 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.797+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:18.850 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.851 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.802+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'mds_autoscaler' 2024-04-02T15:07:18.851 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.802+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'mds_autoscaler': (22) Invalid argument 2024-04-02T15:07:18.851 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.821+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:18.851 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.821+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:18.851 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.851 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.825+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'osd_perf_query' 2024-04-02T15:07:18.851 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.825+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'osd_perf_query': (22) Invalid argument 2024-04-02T15:07:18.851 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.841+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:18.851 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.841+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:18.851 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.851 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.846+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'snap_schedule' 2024-04-02T15:07:18.851 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.846+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'snap_schedule': (22) Invalid argument 2024-04-02T15:07:18.851 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.861+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:18.885 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:18 smithi042 ceph-mon[24805]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:18.886 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:18 smithi042 ceph-mon[29473]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:19.102 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.861+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.103 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.103 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.866+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'dashboard' 2024-04-02T15:07:19.103 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.866+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'dashboard': (22) Invalid argument 2024-04-02T15:07:19.103 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.882+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.103 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.882+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.103 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.103 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.887+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'zabbix' 2024-04-02T15:07:19.103 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.887+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'zabbix': (22) Invalid argument 2024-04-02T15:07:19.103 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.902+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.103 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.902+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.103 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.103 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.907+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'balancer' 2024-04-02T15:07:19.103 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.907+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'balancer': (22) Invalid argument 2024-04-02T15:07:19.104 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.923+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.104 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.923+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.104 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.104 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.927+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'influx' 2024-04-02T15:07:19.104 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.927+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'influx': (22) Invalid argument 2024-04-02T15:07:19.104 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.952+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.104 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.952+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.104 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.104 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.956+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'alerts' 2024-04-02T15:07:19.104 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.957+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'alerts': (22) Invalid argument 2024-04-02T15:07:19.104 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.972+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.105 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.972+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.105 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.105 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.976+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'iostat' 2024-04-02T15:07:19.105 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.976+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'iostat': (22) Invalid argument 2024-04-02T15:07:19.105 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.991+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.105 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.991+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.105 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.105 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.995+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'rgw' 2024-04-02T15:07:19.105 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:18.995+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'rgw': (22) Invalid argument 2024-04-02T15:07:19.105 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.011+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.106 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.011+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.106 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.106 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.015+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'rbd_support' 2024-04-02T15:07:19.106 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.015+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'rbd_support': (22) Invalid argument 2024-04-02T15:07:19.106 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.030+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.106 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.030+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.106 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.106 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.034+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'progress' 2024-04-02T15:07:19.106 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.034+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'progress': (22) Invalid argument 2024-04-02T15:07:19.106 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.049+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.107 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.049+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.107 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.107 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.053+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'pg_autoscaler' 2024-04-02T15:07:19.107 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.053+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'pg_autoscaler': (22) Invalid argument 2024-04-02T15:07:19.107 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.068+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.107 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.068+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.107 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.107 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.073+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'devicehealth' 2024-04-02T15:07:19.107 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.073+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'devicehealth': (22) Invalid argument 2024-04-02T15:07:19.107 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.088+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.107 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.088+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.107 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.107 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.092+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'k8sevents' 2024-04-02T15:07:19.107 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.092+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'k8sevents': (22) Invalid argument 2024-04-02T15:07:19.107 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.109+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.108 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.109+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.108 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.108 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.114+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'rook' 2024-04-02T15:07:19.108 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.114+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'rook': (22) Invalid argument 2024-04-02T15:07:19.403 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.131+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.403 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.131+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.403 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.404 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.136+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'diskprediction_local' 2024-04-02T15:07:19.404 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.136+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'diskprediction_local': (22) Invalid argument 2024-04-02T15:07:19.404 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.179+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.404 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.179+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.404 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.404 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.183+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'mirroring' 2024-04-02T15:07:19.404 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.183+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'mirroring': (22) Invalid argument 2024-04-02T15:07:19.404 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.198+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.404 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.198+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.404 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.404 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.203+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'selftest' 2024-04-02T15:07:19.404 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.203+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'selftest': (22) Invalid argument 2024-04-02T15:07:19.404 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.218+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.405 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.218+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.405 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.405 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.223+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'telegraf' 2024-04-02T15:07:19.405 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.223+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'telegraf': (22) Invalid argument 2024-04-02T15:07:19.405 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.238+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.405 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.238+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.405 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.405 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.242+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'test_orchestrator' 2024-04-02T15:07:19.405 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.242+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'test_orchestrator': (22) Invalid argument 2024-04-02T15:07:19.406 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.257+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.406 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.257+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.406 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.406 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.261+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'localpool' 2024-04-02T15:07:19.406 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.261+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'localpool': (22) Invalid argument 2024-04-02T15:07:19.406 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.277+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.406 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.277+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.406 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.406 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.281+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'crash' 2024-04-02T15:07:19.407 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.281+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'crash': (22) Invalid argument 2024-04-02T15:07:19.407 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.297+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.407 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.297+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.407 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.407 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.301+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'insights' 2024-04-02T15:07:19.407 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.301+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'insights': (22) Invalid argument 2024-04-02T15:07:19.407 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.316+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.407 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.316+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.407 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.407 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.320+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'orchestrator' 2024-04-02T15:07:19.408 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.320+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'orchestrator': (22) Invalid argument 2024-04-02T15:07:19.408 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.336+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.408 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.336+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.408 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.408 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.340+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'osd_support' 2024-04-02T15:07:19.408 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.340+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'osd_support': (22) Invalid argument 2024-04-02T15:07:19.408 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.355+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.408 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.355+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.408 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.408 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.359+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'volumes' 2024-04-02T15:07:19.408 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.359+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'volumes': (22) Invalid argument 2024-04-02T15:07:19.408 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.414+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.725 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.414+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.726 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.726 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.421+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'stats' 2024-04-02T15:07:19.726 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.421+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'stats': (22) Invalid argument 2024-04-02T15:07:19.726 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.440+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.726 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.440+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.726 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.726 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.444+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'cephadm' 2024-04-02T15:07:19.726 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.444+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'cephadm': (22) Invalid argument 2024-04-02T15:07:19.726 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.462+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.726 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.462+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.726 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.726 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.467+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'telemetry' 2024-04-02T15:07:19.726 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.467+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'telemetry': (22) Invalid argument 2024-04-02T15:07:19.726 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.484+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.484+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.488+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'prometheus' 2024-04-02T15:07:19.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.488+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'prometheus': (22) Invalid argument 2024-04-02T15:07:19.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.504+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.504+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.509+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'status' 2024-04-02T15:07:19.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.509+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'status': (22) Invalid argument 2024-04-02T15:07:19.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.528+0000 7fc0d7d8d200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:07:19.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.528+0000 7fc0d7d8d200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:07:19.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.532+0000 7fc0d7d8d200 -1 mgr[py] Class not found in module 'nfs' 2024-04-02T15:07:19.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.532+0000 7fc0d7d8d200 -1 mgr[py] Error loading module 'nfs': (22) Invalid argument 2024-04-02T15:07:19.727 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:07:19.532+0000 7fc0d7d8d200 -1 log_channel(cluster) log [ERR] : Failed to load ceph-mgr modules: restful, mds_autoscaler, osd_perf_query, snap_schedule, dashboard, zabbix, balancer, influx, alerts, iostat, rgw, rbd_support, progress, pg_autoscaler, devicehealth, k8sevents, rook, diskprediction_local, mirroring, selftest, telegraf, test_orchestrator, localpool, crash, insights, orchestrator, osd_support, volumes, stats, cephadm, telemetry, prometheus, status, nfs 2024-04-02T15:07:19.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:07:19.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:07:19.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:07:19.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:07:19.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:07:19.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:07:19.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-mon[25801]: Standby manager daemon x restarted 2024-04-02T15:07:19.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:19 smithi169 ceph-mon[25801]: Standby manager daemon x started 2024-04-02T15:07:20.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:07:19.682Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:07:20.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:07:20.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:07:20.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:07:20.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:07:20.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:07:20.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:07:20.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[24805]: Standby manager daemon x restarted 2024-04-02T15:07:20.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[24805]: Standby manager daemon x started 2024-04-02T15:07:20.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:07:20.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:07:20.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:07:20.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:07:20.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:07:20.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:07:20.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[29473]: Standby manager daemon x restarted 2024-04-02T15:07:20.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:19 smithi042 ceph-mon[29473]: Standby manager daemon x started 2024-04-02T15:07:21.430 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:21 smithi169 ceph-mon[25801]: mgrmap e22: y(active, since 6m), standbys: x 2024-04-02T15:07:21.431 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:21 smithi169 ceph-mon[25801]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:21.541 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:21 smithi042 ceph-mon[24805]: mgrmap e22: y(active, since 6m), standbys: x 2024-04-02T15:07:21.541 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:21 smithi042 ceph-mon[24805]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:21.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:21 smithi042 ceph-mon[29473]: mgrmap e22: y(active, since 6m), standbys: x 2024-04-02T15:07:21.542 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:21 smithi042 ceph-mon[29473]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:22.292 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:07:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:07:21] "GET /metrics HTTP/1.1" 200 207964 "" "Prometheus/2.33.4" 2024-04-02T15:07:22.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:22 smithi169 ceph-mon[25801]: Failed to load ceph-mgr modules: restful, mds_autoscaler, osd_perf_query, snap_schedule, dashboard, zabbix, balancer, influx, alerts, iostat, rgw, rbd_support, progress, pg_autoscaler, devicehealth, k8sevents, rook, diskprediction_local, mirroring, selftest, telegraf, test_orchestrator, localpool, crash, insights, orchestrator, osd_support, volumes, stats, cephadm, telemetry, prometheus, status, nfs 2024-04-02T15:07:22.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:22 smithi042 ceph-mon[24805]: Failed to load ceph-mgr modules: restful, mds_autoscaler, osd_perf_query, snap_schedule, dashboard, zabbix, balancer, influx, alerts, iostat, rgw, rbd_support, progress, pg_autoscaler, devicehealth, k8sevents, rook, diskprediction_local, mirroring, selftest, telegraf, test_orchestrator, localpool, crash, insights, orchestrator, osd_support, volumes, stats, cephadm, telemetry, prometheus, status, nfs 2024-04-02T15:07:22.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:22 smithi042 ceph-mon[29473]: Failed to load ceph-mgr modules: restful, mds_autoscaler, osd_perf_query, snap_schedule, dashboard, zabbix, balancer, influx, alerts, iostat, rgw, rbd_support, progress, pg_autoscaler, devicehealth, k8sevents, rook, diskprediction_local, mirroring, selftest, telegraf, test_orchestrator, localpool, crash, insights, orchestrator, osd_support, volumes, stats, cephadm, telemetry, prometheus, status, nfs 2024-04-02T15:07:23.612 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:23 smithi169 ceph-mon[25801]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:23.612 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:23 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:07:23.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:23 smithi042 ceph-mon[29473]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:23.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:23 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:07:23.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:23 smithi042 ceph-mon[24805]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:23.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:23 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:07:23.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:07:23.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:07:23.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:07:23.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:07:23.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:07:23.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:07:24.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:24 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:07:24.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:24 smithi169 ceph-mon[25801]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:25.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:24 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:07:25.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:24 smithi042 ceph-mon[24805]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:25.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:24 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:07:25.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:24 smithi042 ceph-mon[29473]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:26.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:26 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:26.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:26 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:26.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:26 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:27.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:27 smithi169 ceph-mon[25801]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:27.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:27 smithi042 ceph-mon[24805]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:27.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:27 smithi042 ceph-mon[29473]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:28.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:28 smithi169 ceph-mon[25801]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:29.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:28 smithi042 ceph-mon[24805]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:29.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:28 smithi042 ceph-mon[29473]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:31.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:31 smithi169 ceph-mon[25801]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:31.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:31 smithi042 ceph-mon[24805]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:31.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:31 smithi042 ceph-mon[29473]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:32.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:07:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:07:31] "GET /metrics HTTP/1.1" 200 207964 "" "Prometheus/2.33.4" 2024-04-02T15:07:32.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:32 smithi169 ceph-mon[25801]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:33.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:32 smithi042 ceph-mon[24805]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:33.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:32 smithi042 ceph-mon[29473]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:07:33.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:07:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:07:33.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:07:33.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:07:33.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:07:34.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:34 smithi169 ceph-mon[25801]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:35.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:34 smithi042 ceph-mon[24805]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:35.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:34 smithi042 ceph-mon[29473]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:36.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:36 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:36.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:36 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:36.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:36 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:37.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:37 smithi169 ceph-mon[25801]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:37.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:37 smithi042 ceph-mon[24805]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:37.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:37 smithi042 ceph-mon[29473]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:38.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:38 smithi169 ceph-mon[25801]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:39.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:38 smithi042 ceph-mon[24805]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:39.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:38 smithi042 ceph-mon[29473]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:41.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:41 smithi169 ceph-mon[25801]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:41.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:41 smithi042 ceph-mon[24805]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:41.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:41 smithi042 ceph-mon[29473]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:42.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:07:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:07:41] "GET /metrics HTTP/1.1" 200 207964 "" "Prometheus/2.33.4" 2024-04-02T15:07:42.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:42 smithi169 ceph-mon[25801]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:43.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:42 smithi042 ceph-mon[24805]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:43.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:42 smithi042 ceph-mon[29473]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:43.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:07:43.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:07:43.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:07:43.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:07:43.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:07:43.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:07:44.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:44 smithi169 ceph-mon[25801]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:45.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:44 smithi042 ceph-mon[24805]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:45.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:44 smithi042 ceph-mon[29473]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:46.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:46 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:46.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:46 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:46.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:46 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:47.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:47 smithi169 ceph-mon[25801]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:47.693 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:47 smithi042 ceph-mon[24805]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:47.693 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:47 smithi042 ceph-mon[29473]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:07:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:07:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:07:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:07:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:48 smithi169 ceph-mon[25801]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:49.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:07:49.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:07:49.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:07:49.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:07:49.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:48 smithi042 ceph-mon[24805]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:49.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:07:49.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:07:49.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:07:49.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:07:49.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:48 smithi042 ceph-mon[29473]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:51.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:51 smithi169 ceph-mon[25801]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:51.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:51 smithi042 ceph-mon[24805]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:51.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:51 smithi042 ceph-mon[29473]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:52.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:07:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:07:51] "GET /metrics HTTP/1.1" 200 207964 "" "Prometheus/2.33.4" 2024-04-02T15:07:52.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:52 smithi169 ceph-mon[25801]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:53.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:52 smithi042 ceph-mon[24805]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:53.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:52 smithi042 ceph-mon[29473]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:53.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:07:53.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:07:53.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:07:53.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:07:53.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:07:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:07:53.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:07:54.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:54 smithi169 ceph-mon[25801]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:55.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:54 smithi042 ceph-mon[24805]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:55.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:54 smithi042 ceph-mon[29473]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:07:56.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:56 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:56.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:56 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:56.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:56 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:07:57.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:57 smithi169 ceph-mon[25801]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:57.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:57 smithi042 ceph-mon[24805]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:57.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:57 smithi042 ceph-mon[29473]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:58.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:07:58 smithi169 ceph-mon[25801]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:59.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:07:58 smithi042 ceph-mon[24805]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:07:59.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:07:58 smithi042 ceph-mon[29473]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:01.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:01 smithi169 ceph-mon[25801]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:01.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:01 smithi042 ceph-mon[24805]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:01.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:01 smithi042 ceph-mon[29473]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:02.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:08:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:08:01] "GET /metrics HTTP/1.1" 200 207964 "" "Prometheus/2.33.4" 2024-04-02T15:08:02.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:02 smithi169 ceph-mon[25801]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:03.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:02 smithi042 ceph-mon[24805]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:03.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:02 smithi042 ceph-mon[29473]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:03.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:08:03.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:08:03.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:03.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:08:03.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:03.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:08:04.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:04 smithi169 ceph-mon[25801]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:05.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:04 smithi042 ceph-mon[24805]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:05.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:04 smithi042 ceph-mon[29473]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:06.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:06 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:06.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:06 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:06.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:06 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:07.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:07 smithi169 ceph-mon[25801]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:07.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:07 smithi042 ceph-mon[24805]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:07.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:07 smithi042 ceph-mon[29473]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:08.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:08 smithi169 ceph-mon[25801]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:09.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:08 smithi042 ceph-mon[24805]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:09.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:08 smithi042 ceph-mon[29473]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:11.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:11 smithi169 ceph-mon[25801]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:11.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:11 smithi042 ceph-mon[24805]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:11.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:11 smithi042 ceph-mon[29473]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:12.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:08:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:08:11] "GET /metrics HTTP/1.1" 200 207965 "" "Prometheus/2.33.4" 2024-04-02T15:08:12.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:12 smithi169 ceph-mon[25801]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:13.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:12 smithi042 ceph-mon[24805]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:13.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:12 smithi042 ceph-mon[29473]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:13.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:08:13.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:08:13.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:13.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:08:13.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:13.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:08:14.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:14 smithi169 ceph-mon[25801]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:15.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:14 smithi042 ceph-mon[29473]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:15.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:14 smithi042 ceph-mon[24805]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:16.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:16 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:16.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:16 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:16.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:16 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:17.692 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:17 smithi042 ceph-mon[24805]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:17.692 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:17 smithi042 ceph-mon[29473]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:17.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:17 smithi169 ceph-mon[25801]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:18.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:18 smithi169 ceph-mon[25801]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:19.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:18 smithi042 ceph-mon[24805]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:19.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:18 smithi042 ceph-mon[29473]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:21.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:21 smithi169 ceph-mon[25801]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:21.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:21 smithi042 ceph-mon[24805]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:21.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:21 smithi042 ceph-mon[29473]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:22.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:08:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:08:21] "GET /metrics HTTP/1.1" 200 207958 "" "Prometheus/2.33.4" 2024-04-02T15:08:22.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:22 smithi169 ceph-mon[25801]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:23.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:22 smithi042 ceph-mon[24805]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:23.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:22 smithi042 ceph-mon[29473]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:23.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:23 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:08:23.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:23 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:08:23.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:23 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:08:23.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:23 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:08:23.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:23 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:08:23.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:23 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:08:23.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:08:23.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:08:23.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:23.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:08:23.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:23.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:08:23.956 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:23 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:08:23.956 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:23 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:08:23.956 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:23 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:08:25.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:24 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:08:25.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:24 smithi042 ceph-mon[24805]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:25.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:24 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:08:25.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:24 smithi042 ceph-mon[29473]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:25.225 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:24 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:08:25.226 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:24 smithi169 ceph-mon[25801]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:26.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:26 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:26.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:26 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:26.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:26 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:27.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:27 smithi169 ceph-mon[25801]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:27.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:27 smithi042 ceph-mon[24805]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:27.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:27 smithi042 ceph-mon[29473]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:28.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:28 smithi169 ceph-mon[25801]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:29.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:28 smithi042 ceph-mon[24805]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:29.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:28 smithi042 ceph-mon[29473]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:31.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:31 smithi169 ceph-mon[25801]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:31.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:31 smithi042 ceph-mon[29473]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:31.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:31 smithi042 ceph-mon[24805]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:32.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:08:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:08:31] "GET /metrics HTTP/1.1" 200 207958 "" "Prometheus/2.33.4" 2024-04-02T15:08:32.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:32 smithi169 ceph-mon[25801]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:33.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:32 smithi042 ceph-mon[24805]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:33.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:32 smithi042 ceph-mon[29473]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:08:33.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:08:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:33.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:08:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:33.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:08:34.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:34 smithi169 ceph-mon[25801]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:35.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:34 smithi042 ceph-mon[24805]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:35.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:34 smithi042 ceph-mon[29473]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:36.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:36 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:36.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:36 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:36.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:36 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:37.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:37 smithi169 ceph-mon[25801]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:37.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:37 smithi042 ceph-mon[24805]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:37.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:37 smithi042 ceph-mon[29473]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:38.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:38 smithi169 ceph-mon[25801]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:39.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:38 smithi042 ceph-mon[24805]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:39.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:38 smithi042 ceph-mon[29473]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:41.613 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:41 smithi042 ceph-mon[24805]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:41.613 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:41 smithi042 ceph-mon[29473]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:41.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:41 smithi169 ceph-mon[25801]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:41.868 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:41.626Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:08:41.868 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:41.628Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:08:42.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:08:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:08:41] "GET /metrics HTTP/1.1" 200 207968 "" "Prometheus/2.33.4" 2024-04-02T15:08:42.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:42 smithi169 ceph-mon[25801]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:43.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:42 smithi042 ceph-mon[29473]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:43.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:42 smithi042 ceph-mon[24805]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:43.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:08:43.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:08:43.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:43.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:08:43.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:43.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:08:44.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:44 smithi169 ceph-mon[25801]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:45.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:44 smithi042 ceph-mon[24805]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:45.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:44 smithi042 ceph-mon[29473]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:46.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:46 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:46.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:46 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:46.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:46 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:47.692 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:47 smithi042 ceph-mon[24805]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:47.692 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:47 smithi042 ceph-mon[29473]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:47.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:47 smithi169 ceph-mon[25801]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:48.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:08:48.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:48 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:08:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:08:48.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:48 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:08:49.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:08:49.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:48 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:08:49.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:08:49.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:48 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:08:49.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:08:49.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:48 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:08:49.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:08:49.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:48 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:08:49.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:49 smithi169 ceph-mon[25801]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:50.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:49 smithi042 ceph-mon[29473]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:50.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:49 smithi042 ceph-mon[24805]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:51.613 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:51 smithi042 ceph-mon[24805]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:51.613 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:51 smithi042 ceph-mon[29473]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:51.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:51 smithi169 ceph-mon[25801]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:51.868 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:08:51.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:08:51.868 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:51.627Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:08:51.868 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:51.628Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:08:52.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:08:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:08:51] "GET /metrics HTTP/1.1" 200 207964 "" "Prometheus/2.33.4" 2024-04-02T15:08:52.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:52 smithi169 ceph-mon[25801]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:53.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:52 smithi042 ceph-mon[24805]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:53.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:52 smithi042 ceph-mon[29473]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:53.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:08:53.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:08:53.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:53.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:08:53.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:08:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:08:53.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:08:54.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:54 smithi169 ceph-mon[25801]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:55.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:54 smithi042 ceph-mon[24805]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:55.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:54 smithi042 ceph-mon[29473]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:08:56.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:56 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:56.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:56 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:56.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:56 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:08:57.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:57 smithi169 ceph-mon[25801]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:57.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:57 smithi042 ceph-mon[29473]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:57.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:57 smithi042 ceph-mon[24805]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:58.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:08:58 smithi169 ceph-mon[25801]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:59.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:08:58 smithi042 ceph-mon[24805]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:08:59.042 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:08:58 smithi042 ceph-mon[29473]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:01.613 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:01 smithi042 ceph-mon[24805]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:01.613 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:01 smithi042 ceph-mon[29473]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:01.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:01 smithi169 ceph-mon[25801]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:01.868 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:09:01.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:01.868 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:01.627Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:01.868 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:01.629Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:09:02.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:09:01] "GET /metrics HTTP/1.1" 200 207964 "" "Prometheus/2.33.4" 2024-04-02T15:09:02.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:02 smithi169 ceph-mon[25801]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:03.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:02 smithi042 ceph-mon[29473]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:03.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:02 smithi042 ceph-mon[24805]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:03.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:09:03.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:09:03.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:03.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:03.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:03.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:09:04.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:04 smithi169 ceph-mon[25801]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:05.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:04 smithi042 ceph-mon[24805]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:05.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:04 smithi042 ceph-mon[29473]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:06.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:06 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:06.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:06 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:06.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:06 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:07.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:07 smithi169 ceph-mon[25801]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:07.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:07 smithi042 ceph-mon[24805]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:07.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:07 smithi042 ceph-mon[29473]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:08.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:08 smithi169 ceph-mon[25801]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:09.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:08 smithi042 ceph-mon[24805]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:09.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:08 smithi042 ceph-mon[29473]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:11.295 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch ps' 2024-04-02T15:09:11.541 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:11 smithi042 ceph-mon[24805]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:11.541 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:11 smithi042 ceph-mon[29473]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:11.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:11 smithi169 ceph-mon[25801]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:11.864 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:09:11.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:11.864 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:11.629Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:11.864 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:11.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:09:12.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:09:11] "GET /metrics HTTP/1.1" 200 207966 "" "Prometheus/2.33.4" 2024-04-02T15:09:12.704 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:12 smithi042 ceph-mon[24805]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:12.705 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:12 smithi042 ceph-mon[29473]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:12.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:12 smithi169 ceph-mon[25801]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:13.419 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T15:09:13.419 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (6m) 2m ago 7m 21.8M - ba2b418f427c 64e49f76da9a 2024-04-02T15:09:13.419 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi169 *:3000 running (6m) 110s ago 6m 58.6M - 8.3.5 dad864ee21e9 acd65a3d3b6a 2024-04-02T15:09:13.419 INFO:teuthology.orchestra.run.smithi042.stdout:iscsi.foo.smithi042.toyfgf smithi042 running (6m) 2m ago 6m 48.9M - 3.5 e1d6a67b021e b024dc64805f 2024-04-02T15:09:13.419 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi169 *:8443,9283 running (115s) 110s ago 11m 90.1M - 19.0.0-2592-g3f3f3ed9 7661fe7521b9 4219b532beb6 2024-04-02T15:09:13.419 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:9283 running (13m) 2m ago 13m 461M - 17.2.0 e1d6a67b021e 92a15ec7fb5f 2024-04-02T15:09:13.420 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (13m) 2m ago 13m 50.0M 2048M 17.2.0 e1d6a67b021e d1df5488c219 2024-04-02T15:09:13.420 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi169 running (12m) 110s ago 12m 41.0M 2048M 17.2.0 e1d6a67b021e 1cc495c7e481 2024-04-02T15:09:13.420 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (12m) 2m ago 12m 40.8M 2048M 17.2.0 e1d6a67b021e dbd3308e4000 2024-04-02T15:09:13.420 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (8m) 2m ago 8m 17.6M - 1dbe0e931976 d0a5ca877a2a 2024-04-02T15:09:13.420 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi169 *:9100 running (7m) 110s ago 7m 17.6M - 1dbe0e931976 dbb30e649ccf 2024-04-02T15:09:13.420 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (11m) 2m ago 11m 52.1M 4017M 17.2.0 e1d6a67b021e 590bee772da0 2024-04-02T15:09:13.420 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (11m) 2m ago 11m 50.7M 4017M 17.2.0 e1d6a67b021e 32f3310fd9f9 2024-04-02T15:09:13.420 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (10m) 2m ago 10m 47.0M 4017M 17.2.0 e1d6a67b021e f2d6ce6b9bfa 2024-04-02T15:09:13.420 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (10m) 2m ago 10m 48.7M 4017M 17.2.0 e1d6a67b021e c5d80c30db4c 2024-04-02T15:09:13.420 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi169 running (9m) 110s ago 9m 49.8M 4272M 17.2.0 e1d6a67b021e efc569199c21 2024-04-02T15:09:13.420 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi169 running (9m) 110s ago 9m 47.1M 4272M 17.2.0 e1d6a67b021e d543be94df6d 2024-04-02T15:09:13.420 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi169 running (9m) 110s ago 9m 45.5M 4272M 17.2.0 e1d6a67b021e 34c8a641d867 2024-04-02T15:09:13.420 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi169 running (8m) 110s ago 8m 49.3M 4272M 17.2.0 e1d6a67b021e 817b439db33c 2024-04-02T15:09:13.420 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi169 *:9095 running (6m) 110s ago 8m 67.2M - 514e6a882f6e 80b1a5d12473 2024-04-02T15:09:13.421 INFO:teuthology.orchestra.run.smithi042.stdout:rgw.foo.smithi042.plghvb smithi042 *:8000 running (6m) 2m ago 6m 71.5M - 17.2.0 e1d6a67b021e a220ab61cedb 2024-04-02T15:09:13.421 INFO:teuthology.orchestra.run.smithi042.stdout:rgw.foo.smithi169.pvuzch smithi169 *:8000 running (6m) 110s ago 6m 69.9M - 17.2.0 e1d6a67b021e 7e7d6aa0d138 2024-04-02T15:09:13.547 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:09:13.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:13.547 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:13.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:13.547 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:13.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:09:13.890 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:13 smithi042 ceph-mon[24805]: from='client.15015 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:09:13.891 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:13 smithi042 ceph-mon[29473]: from='client.15015 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:09:13.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:13 smithi169 ceph-mon[25801]: from='client.15015 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:09:14.202 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph versions' 2024-04-02T15:09:14.771 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:14 smithi042 ceph-mon[24805]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:14.771 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:14 smithi042 ceph-mon[29473]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:14.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:14 smithi169 ceph-mon[25801]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:16.357 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-04-02T15:09:16.358 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-04-02T15:09:16.358 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-02T15:09:16.358 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:09:16.358 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-04-02T15:09:16.358 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-02T15:09:16.358 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.0.0-2592-g3f3f3ed9 (3f3f3ed992429b85be729743ad3b39d52dd90744) squid (dev)": 1 2024-04-02T15:09:16.358 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:09:16.358 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-04-02T15:09:16.358 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T15:09:16.358 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:09:16.358 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": {}, 2024-04-02T15:09:16.358 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw": { 2024-04-02T15:09:16.358 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T15:09:16.358 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-04-02T15:09:16.359 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-04-02T15:09:16.359 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-04-02T15:09:16.359 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.0.0-2592-g3f3f3ed9 (3f3f3ed992429b85be729743ad3b39d52dd90744) squid (dev)": 1 2024-04-02T15:09:16.359 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-04-02T15:09:16.359 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-04-02T15:09:16.650 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:16 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:16.651 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:16 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:16.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:16 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:16.959 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph -s' 2024-04-02T15:09:17.521 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:17 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1417497675' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:09:17.521 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:17 smithi042 ceph-mon[24805]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:17.521 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:17 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1417497675' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:09:17.521 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:17 smithi042 ceph-mon[29473]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:17.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:17 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1417497675' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:09:17.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:17 smithi169 ceph-mon[25801]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:18.820 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:18 smithi042 ceph-mon[24805]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:18.820 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:18 smithi042 ceph-mon[29473]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:18.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:18 smithi169 ceph-mon[25801]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:19.094 INFO:teuthology.orchestra.run.smithi042.stdout: cluster: 2024-04-02T15:09:19.094 INFO:teuthology.orchestra.run.smithi042.stdout: id: fd27f2e2-f100-11ee-b647-cb9ed24678a4 2024-04-02T15:09:19.094 INFO:teuthology.orchestra.run.smithi042.stdout: health: HEALTH_OK 2024-04-02T15:09:19.094 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:09:19.094 INFO:teuthology.orchestra.run.smithi042.stdout: services: 2024-04-02T15:09:19.094 INFO:teuthology.orchestra.run.smithi042.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-04-02T15:09:19.095 INFO:teuthology.orchestra.run.smithi042.stdout: mgr: y(active, since 8m), standbys: x 2024-04-02T15:09:19.095 INFO:teuthology.orchestra.run.smithi042.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-04-02T15:09:19.095 INFO:teuthology.orchestra.run.smithi042.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-02T15:09:19.095 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:09:19.095 INFO:teuthology.orchestra.run.smithi042.stdout: data: 2024-04-02T15:09:19.095 INFO:teuthology.orchestra.run.smithi042.stdout: pools: 6 pools, 161 pgs 2024-04-02T15:09:19.095 INFO:teuthology.orchestra.run.smithi042.stdout: objects: 209 objects, 585 KiB 2024-04-02T15:09:19.095 INFO:teuthology.orchestra.run.smithi042.stdout: usage: 68 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:09:19.096 INFO:teuthology.orchestra.run.smithi042.stdout: pgs: 161 active+clean 2024-04-02T15:09:19.096 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:09:19.096 INFO:teuthology.orchestra.run.smithi042.stdout: io: 2024-04-02T15:09:19.096 INFO:teuthology.orchestra.run.smithi042.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-02T15:09:19.096 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-04-02T15:09:19.841 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:19 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1946356625' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T15:09:19.841 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:19 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1946356625' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T15:09:19.868 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph health detail' 2024-04-02T15:09:19.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:19 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1946356625' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T15:09:21.436 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:21 smithi042 ceph-mon[24805]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:21.436 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:21 smithi042 ceph-mon[29473]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:21.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:21 smithi169 ceph-mon[25801]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:21.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:09:21.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:21.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:21.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:21.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:21.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:09:21.910 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-04-02T15:09:22.228 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:09:21] "GET /metrics HTTP/1.1" 200 207959 "" "Prometheus/2.33.4" 2024-04-02T15:09:22.541 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:22 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/1031158764' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T15:09:22.541 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:22 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/1031158764' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T15:09:22.637 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-04-02T15:09:22.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:22 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/1031158764' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T15:09:23.503 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:23 smithi042 ceph-mon[24805]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:23.503 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:23 smithi042 ceph-mon[29473]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:23.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:23 smithi169 ceph-mon[25801]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:23.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:09:23.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:23.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:23.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:23.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:23.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:09:24.465 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:24 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:09:24.465 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:24 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:09:24.465 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:24 smithi042 ceph-mon[24805]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:09:24.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:24 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:09:24.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:24 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:09:24.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:24 smithi042 ceph-mon[29473]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:09:24.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:24 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:09:24.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:24 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:09:24.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:24 smithi169 ceph-mon[25801]: from='mgr.14442 172.21.15.42:0/2856559450' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:09:24.769 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-04-02T15:09:25.401 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph mgr fail' 2024-04-02T15:09:25.713 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:25 smithi042 ceph-mon[24805]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:09:25.714 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:25 smithi042 ceph-mon[24805]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:25.714 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:25 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/618332362' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:09:25.714 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:25 smithi042 ceph-mon[29473]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:09:25.714 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:25 smithi042 ceph-mon[29473]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:25.714 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:25 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/618332362' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:09:25.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:25 smithi169 ceph-mon[25801]: from='mgr.14442 ' entity='mgr.y' 2024-04-02T15:09:25.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:25 smithi169 ceph-mon[25801]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:25.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:25 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/618332362' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:09:26.726 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:26 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:26.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:26 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:26.727 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:26 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:27.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:27 smithi169 ceph-mon[25801]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:27.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:27 smithi042 ceph-mon[24805]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:27.792 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:27 smithi042 ceph-mon[29473]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:28.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:28 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/4083405997' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T15:09:28.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:28 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T15:09:28.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:28 smithi169 ceph-mon[25801]: osdmap e97: 8 total, 8 up, 8 in 2024-04-02T15:09:28.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:28 smithi169 ceph-mon[25801]: Standby manager daemon y started 2024-04-02T15:09:28.729 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:28 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/4083405997' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T15:09:28.729 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:28 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T15:09:28.729 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:28 smithi042 ceph-mon[24805]: osdmap e97: 8 total, 8 up, 8 in 2024-04-02T15:09:28.729 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:28 smithi042 ceph-mon[24805]: Standby manager daemon y started 2024-04-02T15:09:28.729 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ignoring --setuser ceph since I am not root 2024-04-02T15:09:28.729 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ignoring --setgroup ceph since I am not root 2024-04-02T15:09:28.730 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:28 smithi042 ceph-mgr[25052]: -- 172.21.15.42:0/1475815121 <== mon.1 v2:172.21.15.42:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5625b72ce340 con 0x5625b72a6800 2024-04-02T15:09:28.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:28 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/4083405997' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T15:09:28.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:28 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T15:09:28.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:28 smithi042 ceph-mon[29473]: osdmap e97: 8 total, 8 up, 8 in 2024-04-02T15:09:28.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:28 smithi042 ceph-mon[29473]: Standby manager daemon y started 2024-04-02T15:09:29.008 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:28 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:28.930+0000 7f276ded1000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T15:09:29.057 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'sleep 180' 2024-04-02T15:09:29.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:29.021+0000 7f276ded1000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T15:09:29.623 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-02T15:09:29.623 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: mgrmap e23: x(active, starting, since 0.912041s), standbys: y 2024-04-02T15:09:29.623 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:09:29.623 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:09:29.623 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:09:29.623 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T15:09:29.623 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T15:09:29.623 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:09:29.624 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:09:29.624 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:09:29.624 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:09:29.624 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:09:29.624 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:09:29.624 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:09:29.624 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:09:29.624 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T15:09:29.624 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T15:09:29.624 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T15:09:29.624 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[29473]: Manager daemon x is now available 2024-04-02T15:09:29.625 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-02T15:09:29.625 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: mgrmap e23: x(active, starting, since 0.912041s), standbys: y 2024-04-02T15:09:29.625 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:09:29.625 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:09:29.625 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:09:29.625 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T15:09:29.625 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T15:09:29.626 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:09:29.626 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:09:29.626 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:09:29.626 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:09:29.626 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:09:29.626 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:09:29.626 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:09:29.626 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:09:29.626 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T15:09:29.626 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T15:09:29.626 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T15:09:29.627 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-mon[24805]: Manager daemon x is now available 2024-04-02T15:09:29.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-02T15:09:29.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: mgrmap e23: x(active, starting, since 0.912041s), standbys: y 2024-04-02T15:09:29.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:09:29.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:09:29.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:09:29.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T15:09:29.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T15:09:29.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:09:29.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:09:29.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:09:29.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:09:29.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:09:29.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:09:29.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:09:29.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:09:29.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T15:09:29.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T15:09:29.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: from='mgr.24761 172.21.15.169:0/88211408' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T15:09:29.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:29 smithi169 ceph-mon[25801]: Manager daemon x is now available 2024-04-02T15:09:29.970 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:29.653+0000 7f276ded1000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T15:09:29.970 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:29.744+0000 7f276ded1000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T15:09:29.970 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:29.831+0000 7f276ded1000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T15:09:29.970 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:29 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:29.982+0000 7f276ded1000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T15:09:30.257 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:30 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:30.069+0000 7f276ded1000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T15:09:30.257 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:30 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:30.183+0000 7f276ded1000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T15:09:30.257 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:30 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:30.270+0000 7f276ded1000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T15:09:30.523 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:30 smithi042 ceph-mon[24805]: mgrmap e24: x(active, since 1.91239s), standbys: y 2024-04-02T15:09:30.524 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:30 smithi042 ceph-mon[24805]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:09:30.524 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:30 smithi042 ceph-mon[29473]: mgrmap e24: x(active, since 1.91239s), standbys: y 2024-04-02T15:09:30.524 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:30 smithi042 ceph-mon[29473]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:09:30.524 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:30 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:30.448+0000 7f276ded1000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T15:09:30.524 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:30 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:30.536+0000 7f276ded1000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T15:09:30.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:30 smithi169 ceph-mon[25801]: mgrmap e24: x(active, since 1.91239s), standbys: y 2024-04-02T15:09:30.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:30 smithi169 ceph-mon[25801]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:09:30.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:30 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:30.559Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:09:31.715 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:31 smithi042 ceph-mon[24805]: Health check failed: 16 mgr modules have failed dependencies (MGR_MODULE_DEPENDENCY) 2024-04-02T15:09:31.716 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:31 smithi042 ceph-mon[24805]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:09:31.716 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:31 smithi042 ceph-mon[29473]: Health check failed: 16 mgr modules have failed dependencies (MGR_MODULE_DEPENDENCY) 2024-04-02T15:09:31.716 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:31 smithi042 ceph-mon[29473]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:09:31.716 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:31.594+0000 7f276ded1000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T15:09:31.717 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:09:31.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:31.717 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:31.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:09:31.717 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:31.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:31.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:31 smithi169 ceph-mon[25801]: Health check failed: 16 mgr modules have failed dependencies (MGR_MODULE_DEPENDENCY) 2024-04-02T15:09:31.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:31 smithi169 ceph-mon[25801]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:09:32.001 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:31.728+0000 7f276ded1000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T15:09:32.001 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:31.924+0000 7f276ded1000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T15:09:32.002 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:32 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:32.014+0000 7f276ded1000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T15:09:32.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:32 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:32.154+0000 7f276ded1000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:09:32.785 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:32 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:32.481+0000 7f276ded1000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T15:09:32.785 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:32 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:32.710+0000 7f276ded1000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:09:32.785 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:32 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:32.797+0000 7f276ded1000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T15:09:33.041 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:32 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:32.973+0000 7f276ded1000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T15:09:33.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:33 smithi169 ceph-mon[25801]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:09:33.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:33 smithi042 ceph-mon[24805]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:09:33.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:33 smithi042 ceph-mon[29473]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:09:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:09:33.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:33.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:09:33.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:33.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:34.217 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:33.922+0000 7f276ded1000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T15:09:34.473 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:34 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:34.230+0000 7f276ded1000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T15:09:34.473 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:34 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:34.329+0000 7f276ded1000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T15:09:34.473 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:34 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:34.486+0000 7f276ded1000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T15:09:34.782 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:34 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: [02/Apr/2024:15:09:34] ENGINE Bus STARTING 2024-04-02T15:09:34.782 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:34 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: CherryPy Checker: 2024-04-02T15:09:34.783 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:34 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: The Application mounted at '' has an empty config. 2024-04-02T15:09:34.783 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:34 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: 2024-04-02T15:09:34.783 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:34 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: [02/Apr/2024:15:09:34] ENGINE Serving on http://:::9283 2024-04-02T15:09:34.783 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:34 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: [02/Apr/2024:15:09:34] ENGINE Bus STARTED 2024-04-02T15:09:35.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:34 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:34.841Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=3 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:09:35.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:35 smithi169 ceph-mon[25801]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-02T15:09:35.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:35 smithi169 ceph-mon[25801]: Standby manager daemon y restarted 2024-04-02T15:09:35.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:35 smithi169 ceph-mon[25801]: Standby manager daemon y started 2024-04-02T15:09:35.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:35 smithi169 ceph-mon[25801]: from='mgr.? 172.21.15.42:0/960899564' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-02T15:09:35.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:35 smithi169 ceph-mon[25801]: from='mgr.? 172.21.15.42:0/960899564' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T15:09:35.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:35 smithi169 ceph-mon[25801]: from='mgr.? 172.21.15.42:0/960899564' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-02T15:09:35.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:35 smithi169 ceph-mon[25801]: from='mgr.? 172.21.15.42:0/960899564' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T15:09:35.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:35 smithi042 ceph-mon[29473]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-02T15:09:35.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:35 smithi042 ceph-mon[29473]: Standby manager daemon y restarted 2024-04-02T15:09:35.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:35 smithi042 ceph-mon[29473]: Standby manager daemon y started 2024-04-02T15:09:35.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:35 smithi042 ceph-mon[29473]: from='mgr.? 172.21.15.42:0/960899564' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-02T15:09:35.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:35 smithi042 ceph-mon[29473]: from='mgr.? 172.21.15.42:0/960899564' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T15:09:35.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:35 smithi042 ceph-mon[29473]: from='mgr.? 172.21.15.42:0/960899564' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-02T15:09:35.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:35 smithi042 ceph-mon[29473]: from='mgr.? 172.21.15.42:0/960899564' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T15:09:35.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:35 smithi042 ceph-mon[24805]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-04-02T15:09:35.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:35 smithi042 ceph-mon[24805]: Standby manager daemon y restarted 2024-04-02T15:09:35.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:35 smithi042 ceph-mon[24805]: Standby manager daemon y started 2024-04-02T15:09:35.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:35 smithi042 ceph-mon[24805]: from='mgr.? 172.21.15.42:0/960899564' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-02T15:09:35.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:35 smithi042 ceph-mon[24805]: from='mgr.? 172.21.15.42:0/960899564' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T15:09:35.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:35 smithi042 ceph-mon[24805]: from='mgr.? 172.21.15.42:0/960899564' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-02T15:09:35.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:35 smithi042 ceph-mon[24805]: from='mgr.? 172.21.15.42:0/960899564' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T15:09:36.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:36 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:36.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:36 smithi169 ceph-mon[25801]: mgrmap e25: x(active, since 7s), standbys: y 2024-04-02T15:09:36.751 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:36 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:36.751 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:36 smithi042 ceph-mon[29473]: mgrmap e25: x(active, since 7s), standbys: y 2024-04-02T15:09:36.751 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:36 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:36.751 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:36 smithi042 ceph-mon[24805]: mgrmap e25: x(active, since 7s), standbys: y 2024-04-02T15:09:37.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:36 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:36.764Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=5 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:09:37.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:37 smithi169 ceph-mon[25801]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-04-02T15:09:37.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:37 smithi042 ceph-mon[24805]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-04-02T15:09:37.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:37 smithi042 ceph-mon[29473]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-04-02T15:09:39.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:39 smithi169 ceph-mon[25801]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 568 B/s rd, 0 op/s 2024-04-02T15:09:39.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:39 smithi042 ceph-mon[24805]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 568 B/s rd, 0 op/s 2024-04-02T15:09:39.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:39 smithi042 ceph-mon[29473]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 568 B/s rd, 0 op/s 2024-04-02T15:09:41.617 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:41 smithi042 ceph-mon[24805]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 930 B/s rd, 0 op/s 2024-04-02T15:09:41.617 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:41 smithi042 ceph-mon[29473]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 930 B/s rd, 0 op/s 2024-04-02T15:09:41.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:41 smithi169 ceph-mon[25801]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 930 B/s rd, 0 op/s 2024-04-02T15:09:41.868 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:09:41.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:41.868 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:41.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:41.868 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:41.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:09:42.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:09:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:09:43.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:43 smithi169 ceph-mon[25801]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:43.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:43 smithi042 ceph-mon[24805]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:43.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:43 smithi042 ceph-mon[29473]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:43.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:09:43.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:43.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:43.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:43.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:43.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:09:45.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:45 smithi169 ceph-mon[25801]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:45.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:45 smithi042 ceph-mon[24805]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:45.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:45 smithi042 ceph-mon[29473]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:46.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:46 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:46.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:46 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:46.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:46 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:47.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:47 smithi169 ceph-mon[25801]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:47.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:47 smithi042 ceph-mon[24805]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:47.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:47 smithi042 ceph-mon[29473]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:49.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:49 smithi169 ceph-mon[25801]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:49.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:49 smithi042 ceph-mon[24805]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:49.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:49 smithi042 ceph-mon[29473]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:51.617 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:51 smithi042 ceph-mon[24805]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:51.618 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:51 smithi042 ceph-mon[29473]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:51.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:51 smithi169 ceph-mon[25801]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:51.868 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:09:51.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:51.868 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:51.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:51.868 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:51.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:09:52.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:09:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:09:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:09:53.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:53 smithi169 ceph-mon[25801]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:53.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:53 smithi042 ceph-mon[24805]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:53.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:53 smithi042 ceph-mon[29473]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:53.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:09:53.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:53.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:53.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:09:53.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:09:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:09:53.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:09:55.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:55 smithi169 ceph-mon[25801]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:55.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:55 smithi042 ceph-mon[24805]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:55.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:55 smithi042 ceph-mon[29473]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:09:56.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:56 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:56.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:56 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:56.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:56 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:09:57.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:57 smithi169 ceph-mon[25801]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:57.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:57 smithi042 ceph-mon[24805]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:57.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:57 smithi042 ceph-mon[29473]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:59.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:09:59 smithi169 ceph-mon[25801]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:59.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:09:59 smithi042 ceph-mon[24805]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:09:59.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:09:59 smithi042 ceph-mon[29473]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:00.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Health detail: HEALTH_WARN 16 mgr modules have failed dependencies 2024-04-02T15:10:00.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: [WRN] MGR_MODULE_DEPENDENCY: 16 mgr modules have failed dependencies 2024-04-02T15:10:00.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'balancer' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'cephadm' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'crash' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'dashboard' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'devicehealth' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'iostat' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'nfs' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'orchestrator' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'pg_autoscaler' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'progress' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'prometheus' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'rbd_support' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'restful' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'status' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'telemetry' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:00 smithi169 ceph-mon[25801]: Module 'volumes' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Health detail: HEALTH_WARN 16 mgr modules have failed dependencies 2024-04-02T15:10:00.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: [WRN] MGR_MODULE_DEPENDENCY: 16 mgr modules have failed dependencies 2024-04-02T15:10:00.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'balancer' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'cephadm' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'crash' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'dashboard' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'devicehealth' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'iostat' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'nfs' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'orchestrator' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'pg_autoscaler' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'progress' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'prometheus' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'rbd_support' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'restful' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'status' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'telemetry' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[24805]: Module 'volumes' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Health detail: HEALTH_WARN 16 mgr modules have failed dependencies 2024-04-02T15:10:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: [WRN] MGR_MODULE_DEPENDENCY: 16 mgr modules have failed dependencies 2024-04-02T15:10:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'balancer' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'cephadm' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'crash' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'dashboard' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'devicehealth' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'iostat' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'nfs' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'orchestrator' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'pg_autoscaler' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'progress' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'prometheus' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'rbd_support' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'restful' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'status' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'telemetry' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:00.794 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:00 smithi042 ceph-mon[29473]: Module 'volumes' has failed dependency: No module named 'mgr_module' 2024-04-02T15:10:01.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:01 smithi169 ceph-mon[25801]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:01.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:01 smithi042 ceph-mon[24805]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:01.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:01 smithi042 ceph-mon[29473]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:01.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:10:01.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:01.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:01.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:01.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:01.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:02.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:10:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:10:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:10:03.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:03 smithi169 ceph-mon[25801]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:03.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:03 smithi042 ceph-mon[24805]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:03.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:03 smithi042 ceph-mon[29473]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:03.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:10:03.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:03.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:03.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:03.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:03.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:05.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:05 smithi169 ceph-mon[25801]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:05.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:05 smithi042 ceph-mon[24805]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:05.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:05 smithi042 ceph-mon[29473]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:06.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:06 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:06.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:06 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:06.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:06 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:07.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:07 smithi042 ceph-mon[24805]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:07.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:07 smithi042 ceph-mon[29473]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:07.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:07 smithi169 ceph-mon[25801]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:09.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:09 smithi042 ceph-mon[24805]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:09.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:09 smithi042 ceph-mon[29473]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:09.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:09 smithi169 ceph-mon[25801]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:11.619 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:11 smithi042 ceph-mon[24805]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:11.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:11 smithi042 ceph-mon[29473]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:11.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:11 smithi169 ceph-mon[25801]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:12.041 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:10:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:10:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:10:12.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:10:11.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:12.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:11.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:12.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:11.634Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:13.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:13 smithi169 ceph-mon[25801]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:13.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:13 smithi042 ceph-mon[24805]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:13.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:13 smithi042 ceph-mon[29473]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:13.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:10:13.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:13.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:13.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:13.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:13.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:15.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:15 smithi169 ceph-mon[25801]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:15.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:15 smithi042 ceph-mon[24805]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:15.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:15 smithi042 ceph-mon[29473]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:16.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:16 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:16.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:16 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:16.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:16 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:17.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:17 smithi169 ceph-mon[25801]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:17.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:17 smithi042 ceph-mon[24805]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:17.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:17 smithi042 ceph-mon[29473]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:19.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:19 smithi169 ceph-mon[25801]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:19.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:19 smithi042 ceph-mon[24805]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:19.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:19 smithi042 ceph-mon[29473]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:21.619 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:21 smithi042 ceph-mon[24805]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:21.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:21 smithi042 ceph-mon[29473]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:21.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:21 smithi169 ceph-mon[25801]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:22.041 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:10:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:10:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:10:22.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:10:21.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:22.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:21.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:22.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:21.635Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:23.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:23 smithi169 ceph-mon[25801]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:23.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:23 smithi042 ceph-mon[24805]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:23.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:23 smithi042 ceph-mon[29473]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:23.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:10:23.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:23.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:23.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:23.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:23.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:25.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:25 smithi169 ceph-mon[25801]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:25.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:25 smithi042 ceph-mon[24805]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:25.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:25 smithi042 ceph-mon[29473]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:26.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:26 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:26.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:26 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:26.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:26 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:27.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:27 smithi169 ceph-mon[25801]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:27.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:27 smithi042 ceph-mon[24805]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:27.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:27 smithi042 ceph-mon[29473]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:29.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:29 smithi169 ceph-mon[25801]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:29.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:29 smithi042 ceph-mon[24805]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:29.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:29 smithi042 ceph-mon[29473]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:31.620 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:31 smithi042 ceph-mon[24805]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:31.620 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:31 smithi042 ceph-mon[29473]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:31.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:31 smithi169 ceph-mon[25801]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:32.041 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:10:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:10:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:10:32.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:10:31.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:32.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:31.634Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:32.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:31.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:33.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:33 smithi169 ceph-mon[25801]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:33.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:33 smithi042 ceph-mon[24805]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:33.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:33 smithi042 ceph-mon[29473]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:10:33.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:33.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:33.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:35.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:35 smithi169 ceph-mon[25801]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:35.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:35 smithi042 ceph-mon[24805]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:35.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:35 smithi042 ceph-mon[29473]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:36.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:36 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:36.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:36 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:36.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:36 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:37.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:37 smithi169 ceph-mon[25801]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:37.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:37 smithi042 ceph-mon[24805]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:37.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:37 smithi042 ceph-mon[29473]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:39.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:39 smithi042 ceph-mon[29473]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:39.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:39 smithi042 ceph-mon[24805]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:39.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:39 smithi169 ceph-mon[25801]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:41.620 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:41 smithi042 ceph-mon[24805]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:41.620 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:41 smithi042 ceph-mon[29473]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:41.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:41 smithi169 ceph-mon[25801]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:42.041 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:10:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:10:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:10:42.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:10:41.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:42.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:41.634Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:42.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:41.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:43.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:43 smithi169 ceph-mon[25801]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:43.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:43 smithi042 ceph-mon[24805]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:43.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:43 smithi042 ceph-mon[29473]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:43.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:10:43.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:43.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:43.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:43.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:43.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:45.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:45 smithi169 ceph-mon[25801]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:45.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:45 smithi042 ceph-mon[24805]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:45.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:45 smithi042 ceph-mon[29473]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:46.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:46 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:46.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:46 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:46.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:46 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:47.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:47 smithi042 ceph-mon[24805]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:47.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:47 smithi042 ceph-mon[29473]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:47.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:47 smithi169 ceph-mon[25801]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:49.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:49 smithi042 ceph-mon[24805]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:49.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:49 smithi042 ceph-mon[29473]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:49.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:49 smithi169 ceph-mon[25801]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:51.621 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:51 smithi042 ceph-mon[24805]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:51.622 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:51 smithi042 ceph-mon[29473]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:51.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:51 smithi169 ceph-mon[25801]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:52.041 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:10:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:10:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:10:52.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:10:51.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:52.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:51.635Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:52.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:51.637Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:53.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:53 smithi169 ceph-mon[25801]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:53.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:53 smithi042 ceph-mon[24805]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:53.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:53 smithi042 ceph-mon[29473]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:53.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:10:53.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:53.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:53.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:10:53.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:10:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:10:53.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:10:55.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:55 smithi042 ceph-mon[24805]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:55.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:55 smithi042 ceph-mon[29473]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:55.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:55 smithi169 ceph-mon[25801]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:10:56.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:56 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:56.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:56 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:56.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:56 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:10:57.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:57 smithi042 ceph-mon[24805]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:57.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:57 smithi042 ceph-mon[29473]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:57.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:57 smithi169 ceph-mon[25801]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:59.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:10:59 smithi042 ceph-mon[24805]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:59.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:10:59 smithi042 ceph-mon[29473]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:10:59.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:10:59 smithi169 ceph-mon[25801]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:01.622 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:01 smithi042 ceph-mon[29473]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:01.622 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:01 smithi042 ceph-mon[24805]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:01.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:01 smithi169 ceph-mon[25801]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:02.040 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:11:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:11:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:11:02.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:11:01.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:02.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:01.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:02.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:01.638Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:03.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:03 smithi042 ceph-mon[24805]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:03.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:03 smithi042 ceph-mon[29473]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:03.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:11:03.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:03.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:03.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:03.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:03.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:03.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:03 smithi169 ceph-mon[25801]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:05.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:05 smithi042 ceph-mon[24805]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:05.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:05 smithi042 ceph-mon[29473]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:05.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:05 smithi169 ceph-mon[25801]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:06.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:06 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:06.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:06 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:06.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:06 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:07.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:07 smithi042 ceph-mon[24805]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:07.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:07 smithi042 ceph-mon[29473]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:07.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:07 smithi169 ceph-mon[25801]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:09.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:09 smithi042 ceph-mon[24805]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:09.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:09 smithi042 ceph-mon[29473]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:09.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:09 smithi169 ceph-mon[25801]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:11.623 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:11 smithi042 ceph-mon[24805]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:11.623 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:11 smithi042 ceph-mon[29473]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:11.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:11 smithi169 ceph-mon[25801]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:12.041 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:11:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:11:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:11:12.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:11:11.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:12.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:11.637Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:12.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:11.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:13.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:13 smithi042 ceph-mon[24805]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:13.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:13 smithi042 ceph-mon[29473]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:13.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:11:13.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:13.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:13.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:13.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:13.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:13.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:13 smithi169 ceph-mon[25801]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:15.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:15 smithi042 ceph-mon[29473]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:15.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:15 smithi042 ceph-mon[24805]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:15.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:15 smithi169 ceph-mon[25801]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:16.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:16 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:16.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:16 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:16.876 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:16 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:17.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:17 smithi042 ceph-mon[24805]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:17.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:17 smithi042 ceph-mon[29473]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:17.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:17 smithi169 ceph-mon[25801]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:19.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:19 smithi042 ceph-mon[24805]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:19.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:19 smithi042 ceph-mon[29473]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:19.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:19 smithi169 ceph-mon[25801]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:21.623 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:21 smithi042 ceph-mon[24805]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:21.623 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:21 smithi042 ceph-mon[29473]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:21.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:21 smithi169 ceph-mon[25801]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:22.041 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:11:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:11:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:11:22.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:11:21.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:22.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:21.638Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:22.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:21.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:23.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:23 smithi042 ceph-mon[24805]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:23.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:23 smithi042 ceph-mon[29473]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:23.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:11:23.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:23.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:23.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:23.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:23.525Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:23.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:23 smithi169 ceph-mon[25801]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:25.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:25 smithi042 ceph-mon[24805]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:25.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:25 smithi042 ceph-mon[29473]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:25.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:25 smithi169 ceph-mon[25801]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:26.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:26 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:26.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:26 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:26.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:26 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:27.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:27 smithi042 ceph-mon[24805]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:27.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:27 smithi042 ceph-mon[29473]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:27.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:27 smithi169 ceph-mon[25801]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:29.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:29 smithi042 ceph-mon[24805]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:29.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:29 smithi042 ceph-mon[29473]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:29.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:29 smithi169 ceph-mon[25801]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:31.623 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:31 smithi042 ceph-mon[29473]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:31.624 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:31 smithi042 ceph-mon[24805]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:31.725 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:31 smithi169 ceph-mon[25801]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:32.041 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:11:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:11:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:11:32.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:11:31.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:32.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:31.638Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:32.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:31.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:33.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:33 smithi042 ceph-mon[24805]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:33.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:33 smithi042 ceph-mon[29473]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:11:33.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:33.525Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:33.526Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:33.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:33 smithi169 ceph-mon[25801]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:35.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:35 smithi042 ceph-mon[24805]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:35.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:35 smithi042 ceph-mon[29473]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:35.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:35 smithi169 ceph-mon[25801]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:36.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:36 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:36.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:36 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:36.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:36 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:37.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:37 smithi042 ceph-mon[24805]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:37.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:37 smithi042 ceph-mon[29473]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:37.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:37 smithi169 ceph-mon[25801]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:39.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:39 smithi042 ceph-mon[24805]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:39.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:39 smithi042 ceph-mon[29473]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:39.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:39 smithi169 ceph-mon[25801]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:41.625 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:41 smithi042 ceph-mon[24805]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:41.625 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:41 smithi042 ceph-mon[29473]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:41.724 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:41 smithi169 ceph-mon[25801]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:42.040 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:11:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:11:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:11:42.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:11:41.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:42.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:41.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:42.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:41.641Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:43.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:43 smithi042 ceph-mon[24805]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:43.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:43 smithi042 ceph-mon[29473]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:43.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:11:43.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:43.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:43.526Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:43.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:43.527Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:43.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:43 smithi169 ceph-mon[25801]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:45.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:45 smithi042 ceph-mon[29473]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:45.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:45 smithi042 ceph-mon[24805]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:45.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:45 smithi169 ceph-mon[25801]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:46.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:46 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:46.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:46 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:46.875 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:46 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:47.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:47 smithi042 ceph-mon[29473]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:47.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:47 smithi042 ceph-mon[24805]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:47.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:47 smithi169 ceph-mon[25801]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:49.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:49 smithi042 ceph-mon[29473]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:49.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:49 smithi042 ceph-mon[24805]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:49.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:49 smithi169 ceph-mon[25801]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:51.625 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:51 smithi042 ceph-mon[24805]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:51.626 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:51 smithi042 ceph-mon[29473]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:51.724 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:51 smithi169 ceph-mon[25801]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:52.041 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:11:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:11:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:11:52.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:11:51.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:52.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:51.640Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:52.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:51.641Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:53.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:53 smithi042 ceph-mon[24805]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:53.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:53 smithi042 ceph-mon[29473]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:53.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:11:53.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:53.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:53.527Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:11:53.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:11:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:11:53.528Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:11:53.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:53 smithi169 ceph-mon[25801]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:55.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:55 smithi042 ceph-mon[24805]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:55.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:55 smithi042 ceph-mon[29473]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:55.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:55 smithi169 ceph-mon[25801]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:11:56.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:56 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:56.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:56 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:56.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:56 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:11:57.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:57 smithi042 ceph-mon[24805]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:57.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:57 smithi042 ceph-mon[29473]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:57.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:57 smithi169 ceph-mon[25801]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:59.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:11:59 smithi042 ceph-mon[24805]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:59.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:11:59 smithi042 ceph-mon[29473]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:11:59.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:11:59 smithi169 ceph-mon[25801]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:01.626 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:01 smithi042 ceph-mon[24805]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:01.626 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:01 smithi042 ceph-mon[29473]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:01.724 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:01 smithi169 ceph-mon[25801]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:02.040 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:12:01.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:12:02.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:01.641Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:02.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:01.642Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:12:02.041 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:12:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:12:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:12:03.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:03 smithi042 ceph-mon[24805]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:03.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:03 smithi042 ceph-mon[29473]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:03.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:12:03.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:03.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:03.528Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:03.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:03.529Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:12:03.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:03 smithi169 ceph-mon[25801]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:05.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:05 smithi042 ceph-mon[29473]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:05.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:05 smithi042 ceph-mon[24805]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:05.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:05 smithi169 ceph-mon[25801]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:06.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:06 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:12:06.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:06 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:12:06.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:06 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:12:07.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:07 smithi042 ceph-mon[29473]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:07.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:07 smithi042 ceph-mon[24805]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:07.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:07 smithi169 ceph-mon[25801]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:09.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:09 smithi042 ceph-mon[24805]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:09.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:09 smithi042 ceph-mon[29473]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:09.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:09 smithi169 ceph-mon[25801]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:11.627 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:11 smithi042 ceph-mon[24805]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:11.628 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:11 smithi042 ceph-mon[29473]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:11.724 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:11 smithi169 ceph-mon[25801]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:12.040 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:12:11.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:12.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:11.642Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:12.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:11.644Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:12:12.041 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:12:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:12:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:12:13.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:13 smithi042 ceph-mon[24805]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:13.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:13 smithi042 ceph-mon[29473]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:13.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:12:13.528Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:12:13.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:13.529Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:13.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:13.530Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:12:13.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:13 smithi169 ceph-mon[25801]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:15.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:15 smithi042 ceph-mon[24805]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:15.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:15 smithi042 ceph-mon[29473]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:15.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:15 smithi169 ceph-mon[25801]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:16.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:16 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:12:16.791 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:16 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:12:16.876 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:16 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:12:17.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:17 smithi042 ceph-mon[24805]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:17.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:17 smithi042 ceph-mon[29473]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:17.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:17 smithi169 ceph-mon[25801]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:19.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:19 smithi042 ceph-mon[24805]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:19.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:19 smithi042 ceph-mon[29473]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:19.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:19 smithi169 ceph-mon[25801]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:21.627 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:21 smithi042 ceph-mon[24805]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:21.628 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:21 smithi042 ceph-mon[29473]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:21.724 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:21 smithi169 ceph-mon[25801]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:22.040 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:12:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:12:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:12:22.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:12:21.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:22.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:21.642Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:22.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:21.644Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:12:23.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:23 smithi042 ceph-mon[24805]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:23.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:23 smithi042 ceph-mon[29473]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:23.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:12:23.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:23.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:23.530Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:23.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:23.531Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:12:23.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:23 smithi169 ceph-mon[25801]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:25.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:25 smithi042 ceph-mon[24805]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:25.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:25 smithi042 ceph-mon[29473]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:25.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:25 smithi169 ceph-mon[25801]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:26.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:26 smithi042 ceph-mon[24805]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:12:26.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:26 smithi042 ceph-mon[29473]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:12:26.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:26 smithi169 ceph-mon[25801]: from='client.24733 -' entity='client.iscsi.foo.smithi042.toyfgf' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:12:27.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:27 smithi042 ceph-mon[24805]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:27.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:27 smithi042 ceph-mon[29473]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:27.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:27 smithi169 ceph-mon[25801]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:29.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:29 smithi042 ceph-mon[24805]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:29.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:29 smithi042 ceph-mon[29473]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:29.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:29 smithi169 ceph-mon[25801]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:31.434 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-02T15:12:31.629 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:31 smithi042 ceph-mon[24805]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:31.629 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:31 smithi042 ceph-mon[29473]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:31.724 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:31 smithi169 ceph-mon[25801]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:12:31.957 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:12:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y[25048]: ::ffff:172.21.15.169 - - [02/Apr/2024:15:12:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:12:31.957 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:12:31.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:12:31.957 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:31.643Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:31.957 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:31.645Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:12:33.685 INFO:teuthology.orchestra.run.smithi042.stderr:no valid command found; 10 closest matches: 2024-04-02T15:12:33.685 INFO:teuthology.orchestra.run.smithi042.stderr:pg stat 2024-04-02T15:12:33.685 INFO:teuthology.orchestra.run.smithi042.stderr:pg getmap 2024-04-02T15:12:33.685 INFO:teuthology.orchestra.run.smithi042.stderr:pg dump [...] 2024-04-02T15:12:33.685 INFO:teuthology.orchestra.run.smithi042.stderr:pg dump_json [...] 2024-04-02T15:12:33.686 INFO:teuthology.orchestra.run.smithi042.stderr:pg dump_pools_json 2024-04-02T15:12:33.686 INFO:teuthology.orchestra.run.smithi042.stderr:pg ls-by-pool [...] 2024-04-02T15:12:33.686 INFO:teuthology.orchestra.run.smithi042.stderr:pg ls-by-primary [] [...] 2024-04-02T15:12:33.686 INFO:teuthology.orchestra.run.smithi042.stderr:pg ls-by-osd [] [...] 2024-04-02T15:12:33.686 INFO:teuthology.orchestra.run.smithi042.stderr:pg ls [] [...] 2024-04-02T15:12:33.686 INFO:teuthology.orchestra.run.smithi042.stderr:pg dump_stuck [...] [] 2024-04-02T15:12:33.686 INFO:teuthology.orchestra.run.smithi042.stderr:Error EINVAL: invalid command 2024-04-02T15:12:33.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:33 smithi042 ceph-mon[24805]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:33.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:33 smithi042 ceph-mon[24805]: from='client.? 172.21.15.42:0/4124149311' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:12:33.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:33 smithi042 ceph-mon[29473]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:33.791 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:33 smithi042 ceph-mon[29473]: from='client.? 172.21.15.42:0/4124149311' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:12:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:12:33.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:33.531Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:33.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.42 because it doesn't contain any IP SANs" 2024-04-02T15:12:33.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:33 smithi169 ceph-mon[25801]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:12:33.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:33 smithi169 ceph-mon[25801]: from='client.? 172.21.15.42:0/4124149311' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:12:34.440 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-04-02T15:12:34.441 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa/tasks/cephadm.py", line 1459, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi042 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c \'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1\'' 2024-04-02T15:12:34.655 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=f22d0fd28a53454f9bd4c885fbe87321 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa/tasks/cephadm.py", line 1459, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi042 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c \'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1\'' 2024-04-02T15:12:34.658 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-02T15:12:34.669 INFO:tasks.cephadm:Teardown begin 2024-04-02T15:12:34.670 DEBUG:teuthology.orchestra.run.smithi042:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T15:12:34.703 DEBUG:teuthology.orchestra.run.smithi169:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T15:12:34.739 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-02T15:12:34.739 DEBUG:teuthology.orchestra.run.smithi042:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-02T15:12:34.757 DEBUG:teuthology.orchestra.run.smithi169:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-02T15:12:34.795 INFO:tasks.cephadm:Stopping all daemons... 2024-04-02T15:12:34.796 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-02T15:12:34.796 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mon.a 2024-04-02T15:12:35.004 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:34 smithi042 systemd[1]: Stopping Ceph mon.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:12:35.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:34 smithi042 bash[56913]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon.a" found: no such container 2024-04-02T15:12:35.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:35 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-a[24800]: 2024-04-02T15:12:35.018+0000 7fbcba3cf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:12:35.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:35 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-a[24800]: 2024-04-02T15:12:35.018+0000 7fbcba3cf700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-02T15:12:35.291 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:35 smithi042 podman[56927]: 2024-04-02 15:12:35.039545344 +0000 UTC m=+0.143941538 container died d1df5488c2195e8e23279790235f6302ddeecd52133da084e02e26feeb77f278 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.buildah.version=1.19.8, ceph=True, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, vcs-type=git, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, maintainer=Guillaume Abrioux , architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream) 2024-04-02T15:12:35.655 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:35 smithi042 podman[56927]: 2024-04-02 15:12:35.410419657 +0000 UTC m=+0.514815838 container cleanup d1df5488c2195e8e23279790235f6302ddeecd52133da084e02e26feeb77f278 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-a, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, GIT_CLEAN=True, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.buildah.version=1.19.8, ceph=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, version=8, vendor=Red Hat, Inc., name=centos-stream, release=754) 2024-04-02T15:12:35.656 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:35 smithi042 bash[56927]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-a 2024-04-02T15:12:35.656 INFO:journalctl@ceph.mon.a.smithi042.stdout:Apr 02 15:12:35 smithi042 bash[56956]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon.a" found: no such container 2024-04-02T15:12:35.660 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mon.a.service' 2024-04-02T15:12:35.711 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:12:35.711 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-02T15:12:35.712 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-02T15:12:35.712 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mon.c 2024-04-02T15:12:36.040 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:35 smithi042 systemd[1]: Stopping Ceph mon.c for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:12:36.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:35 smithi042 bash[57009]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon.c" found: no such container 2024-04-02T15:12:36.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:35 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-c[29469]: 2024-04-02T15:12:35.864+0000 7f38bdcc8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:12:36.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:35 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-c[29469]: 2024-04-02T15:12:35.864+0000 7f38bdcc8700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-04-02T15:12:36.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:35 smithi042 podman[57021]: 2024-04-02 15:12:35.916152589 +0000 UTC m=+0.120955480 container died dbd3308e4000dcbf58ced958a16d065de3eee26554eebe6298cef2f8e558c127 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-c, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., name=centos-stream, architecture=x86_64, release=754, version=8, RELEASE=HEAD, vcs-type=git, ceph=True, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-02T15:12:36.418 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:36 smithi042 podman[57021]: 2024-04-02 15:12:36.243547028 +0000 UTC m=+0.448349913 container cleanup dbd3308e4000dcbf58ced958a16d065de3eee26554eebe6298cef2f8e558c127 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-c, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, release=754, com.redhat.component=centos-stream-container, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, name=centos-stream, architecture=x86_64, vcs-type=git) 2024-04-02T15:12:36.418 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:36 smithi042 bash[57021]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-c 2024-04-02T15:12:36.419 INFO:journalctl@ceph.mon.c.smithi042.stdout:Apr 02 15:12:36 smithi042 bash[57048]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon.c" found: no such container 2024-04-02T15:12:36.423 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mon.c.service' 2024-04-02T15:12:36.460 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:12:36.460 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-02T15:12:36.461 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-02T15:12:36.461 DEBUG:teuthology.orchestra.run.smithi169:> sudo systemctl stop ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mon.b 2024-04-02T15:12:36.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:36 smithi169 systemd[1]: Stopping Ceph mon.b for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:12:36.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:36 smithi169 bash[46928]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon.b" found: no such container 2024-04-02T15:12:36.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:36 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-b[25783]: 2024-04-02T15:12:36.671+0000 7fe4aeaf3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:12:36.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:36 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-b[25783]: 2024-04-02T15:12:36.671+0000 7fe4aeaf3700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-04-02T15:12:36.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:36 smithi169 podman[46941]: 2024-04-02 15:12:36.733403696 +0000 UTC m=+0.166597906 container died 1cc495c7e481ad4e309d21d89c2dc8a88cd3a7b202fe7e8f407a515ed9112e37 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-b, RELEASE=HEAD, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.component=centos-stream-container, vcs-type=git, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, name=centos-stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, vendor=Red Hat, Inc., architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-02T15:12:37.351 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:37 smithi169 podman[46941]: 2024-04-02 15:12:37.172582131 +0000 UTC m=+0.605776383 container cleanup 1cc495c7e481ad4e309d21d89c2dc8a88cd3a7b202fe7e8f407a515ed9112e37 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-b, architecture=x86_64, io.buildah.version=1.19.8, name=centos-stream, RELEASE=HEAD, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , release=754, com.redhat.component=centos-stream-container, io.openshift.expose-services=, vcs-type=git, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, build-date=2022-05-03T08:36:31.336870) 2024-04-02T15:12:37.351 INFO:journalctl@ceph.mon.b.smithi169.stdout:Apr 02 15:12:37 smithi169 bash[46941]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mon-b 2024-04-02T15:12:37.383 DEBUG:teuthology.orchestra.run.smithi169:> sudo pkill -f 'journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mon.b.service' 2024-04-02T15:12:37.435 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:12:37.435 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-02T15:12:37.435 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-02T15:12:37.436 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mgr.y 2024-04-02T15:12:37.790 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:12:37 smithi042 systemd[1]: Stopping Ceph mgr.y for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:12:37.791 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:12:37 smithi042 bash[57100]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr.y" found: no such container 2024-04-02T15:12:37.791 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:12:37 smithi042 podman[57113]: 2024-04-02 15:12:37.675262865 +0000 UTC m=+0.146914146 container died 92a15ec7fb5f99b61368b9326d37fdd66955da4d42d1f6254040003f84d434b6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y, architecture=x86_64, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=-17.2.0, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.openshift.expose-services=, GIT_CLEAN=True, release=754, GIT_BRANCH=HEAD, RELEASE=HEAD) 2024-04-02T15:12:38.290 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:12:38 smithi042 podman[57113]: 2024-04-02 15:12:38.060495738 +0000 UTC m=+0.532147018 container cleanup 92a15ec7fb5f99b61368b9326d37fdd66955da4d42d1f6254040003f84d434b6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, name=centos-stream, vcs-type=git, version=8, architecture=x86_64, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, RELEASE=HEAD, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, maintainer=Guillaume Abrioux ) 2024-04-02T15:12:38.290 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:12:38 smithi042 bash[57113]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-y 2024-04-02T15:12:38.291 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Apr 02 15:12:38 smithi042 bash[57141]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr.y" found: no such container 2024-04-02T15:12:38.328 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mgr.y.service' 2024-04-02T15:12:38.369 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:12:38.369 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-02T15:12:38.370 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-02T15:12:38.370 DEBUG:teuthology.orchestra.run.smithi169:> sudo systemctl stop ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mgr.x 2024-04-02T15:12:38.724 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:12:38 smithi169 systemd[1]: Stopping Ceph mgr.x for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:12:38.725 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:12:38 smithi169 bash[47023]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr.x" found: no such container 2024-04-02T15:12:38.725 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:12:38 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:12:38.576+0000 7fc0cc37b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.x -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:12:38.725 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:12:38 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x[46023]: 2024-04-02T15:12:38.576+0000 7fc0cc37b700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-02T15:12:38.725 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:12:38 smithi169 podman[47035]: 2024-04-02 15:12:38.587662207 +0000 UTC m=+0.119875657 container died 4219b532beb653631b6eb5f1cf16a3c843222d10c8f5f06c7432cc6855a23f19 (image=quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-04-02-0703-3f3f3ed, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, ceph=True) 2024-04-02T15:12:38.790 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:38 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:38.469Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:12:39.089 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:12:38 smithi169 podman[47035]: 2024-04-02 15:12:38.902758187 +0000 UTC m=+0.434971637 container cleanup 4219b532beb653631b6eb5f1cf16a3c843222d10c8f5f06c7432cc6855a23f19 (image=quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-04-02-0703-3f3f3ed, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T15:12:39.089 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:12:38 smithi169 bash[47035]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr-x 2024-04-02T15:12:39.090 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Apr 02 15:12:38 smithi169 bash[47061]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-mgr.x" found: no such container 2024-04-02T15:12:39.094 DEBUG:teuthology.orchestra.run.smithi169:> sudo pkill -f 'journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@mgr.x.service' 2024-04-02T15:12:39.133 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:12:39.133 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-02T15:12:39.133 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-02T15:12:39.133 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.0 2024-04-02T15:12:39.454 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:39 smithi042 systemd[1]: Stopping Ceph osd.0 for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:12:39.454 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:39 smithi042 bash[57193]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.0" found: no such container 2024-04-02T15:12:39.454 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:39 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-0[32322]: 2024-04-02T15:12:39.355+0000 7fdaebbc4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:12:39.455 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:39 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-0[32322]: 2024-04-02T15:12:39.355+0000 7fdaebbc4700 -1 osd.0 97 *** Got signal Terminated *** 2024-04-02T15:12:39.455 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:39 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-0[32322]: 2024-04-02T15:12:39.355+0000 7fdaebbc4700 -1 osd.0 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:12:39.790 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:39 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:39.468Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:12:42.040 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:12:41.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:42.040 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:41.644Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:12:42.040 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:41.644Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:43.790 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:12:43.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:12:43.790 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:43.531Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:12:43.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:43.531Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:45.009 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:44 smithi042 podman[57206]: 2024-04-02 15:12:44.757214418 +0000 UTC m=+5.528592983 container died 590bee772da0c55fa7a4cc9f0040912cec6ab0d6e5ab3e116b48ec953f3ccb14 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, release=754, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git) 2024-04-02T15:12:45.293 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:45 smithi042 podman[57206]: 2024-04-02 15:12:45.114634634 +0000 UTC m=+5.886013194 container cleanup 590bee772da0c55fa7a4cc9f0040912cec6ab0d6e5ab3e116b48ec953f3ccb14 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-0, release=754, build-date=2022-05-03T08:36:31.336870, ceph=True, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, version=8, GIT_CLEAN=True, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git) 2024-04-02T15:12:45.293 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:45 smithi042 bash[57206]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-0 2024-04-02T15:12:45.293 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:45 smithi042 bash[57239]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.0" found: no such container 2024-04-02T15:12:45.711 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:45 smithi042 podman[57225]: 2024-04-02 15:12:45.40773751 +0000 UTC m=+0.643874927 container remove 590bee772da0c55fa7a4cc9f0040912cec6ab0d6e5ab3e116b48ec953f3ccb14 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-0, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, name=centos-stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, distribution-scope=public, io.openshift.expose-services=, vcs-type=git) 2024-04-02T15:12:45.711 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:45 smithi042 podman[57301]: 2024-04-02 15:12:45.50391246 +0000 UTC m=+0.018569626 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:12:45.712 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:45 smithi042 podman[57301]: 2024-04-02 15:12:45.617056239 +0000 UTC m=+0.131713467 container create 14ccb406b782e9f9d27ffef1d192c46c73a85cd9af6bb972c6b642f5022b3f93 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-0-deactivate, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, version=8, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, name=centos-stream, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-02T15:12:46.040 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:45 smithi042 podman[57301]: 2024-04-02 15:12:45.836107096 +0000 UTC m=+0.350764264 container init 14ccb406b782e9f9d27ffef1d192c46c73a85cd9af6bb972c6b642f5022b3f93 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-0-deactivate, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, distribution-scope=public, release=754, architecture=x86_64, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_CLEAN=True, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, RELEASE=HEAD, version=8, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-02T15:12:46.040 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:45 smithi042 podman[57301]: 2024-04-02 15:12:45.839244669 +0000 UTC m=+0.353901833 container start 14ccb406b782e9f9d27ffef1d192c46c73a85cd9af6bb972c6b642f5022b3f93 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-0-deactivate, io.buildah.version=1.19.8, RELEASE=HEAD, version=8, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.openshift.expose-services=, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, name=centos-stream, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD) 2024-04-02T15:12:46.040 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:45 smithi042 podman[57301]: 2024-04-02 15:12:45.935857551 +0000 UTC m=+0.450514709 container attach 14ccb406b782e9f9d27ffef1d192c46c73a85cd9af6bb972c6b642f5022b3f93 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-0-deactivate, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.openshift.expose-services=, maintainer=Guillaume Abrioux , version=8, release=754, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, distribution-scope=public, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, vcs-type=git) 2024-04-02T15:12:46.416 INFO:journalctl@ceph.osd.0.smithi042.stdout:Apr 02 15:12:46 smithi042 podman[57301]: 2024-04-02 15:12:46.12302106 +0000 UTC m=+0.637678219 container died 14ccb406b782e9f9d27ffef1d192c46c73a85cd9af6bb972c6b642f5022b3f93 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-0-deactivate, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, maintainer=Guillaume Abrioux , version=8, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, architecture=x86_64, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-02T15:12:46.729 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.0.service' 2024-04-02T15:12:46.772 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:12:46.772 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-02T15:12:46.772 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-02T15:12:46.773 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.1 2024-04-02T15:12:47.003 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:46 smithi042 systemd[1]: Stopping Ceph osd.1 for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:12:47.004 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:46 smithi042 bash[57378]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.1" found: no such container 2024-04-02T15:12:47.290 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:47 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-1[35300]: 2024-04-02T15:12:47.017+0000 7f082d544700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:12:47.290 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:47 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-1[35300]: 2024-04-02T15:12:47.017+0000 7f082d544700 -1 osd.1 97 *** Got signal Terminated *** 2024-04-02T15:12:47.290 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:47 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-1[35300]: 2024-04-02T15:12:47.017+0000 7f082d544700 -1 osd.1 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:12:52.040 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:12:51.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:12:52.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:51.645Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:12:52.041 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:51 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:51.645Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:52.813 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:52 smithi042 podman[57391]: 2024-04-02 15:12:52.561019101 +0000 UTC m=+5.652148650 container died 32f3310fd9f9f12eadfe06c4536b8e793a9e51cff57a9e319ddc06c441eb31cf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-1, io.openshift.expose-services=, maintainer=Guillaume Abrioux , RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, ceph=True, vcs-type=git, io.k8s.display-name=CentOS Stream 8, version=8, name=centos-stream, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream) 2024-04-02T15:12:53.145 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:52 smithi042 podman[57391]: 2024-04-02 15:12:52.933784239 +0000 UTC m=+6.024913768 container cleanup 32f3310fd9f9f12eadfe06c4536b8e793a9e51cff57a9e319ddc06c441eb31cf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-1, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, version=8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, release=754, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=) 2024-04-02T15:12:53.145 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:52 smithi042 bash[57391]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-1 2024-04-02T15:12:53.145 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:52 smithi042 bash[57434]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.1" found: no such container 2024-04-02T15:12:53.453 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:53 smithi042 podman[57421]: 2024-04-02 15:12:53.159807479 +0000 UTC m=+0.597000166 container remove 32f3310fd9f9f12eadfe06c4536b8e793a9e51cff57a9e319ddc06c441eb31cf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-1, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , io.openshift.expose-services=, vendor=Red Hat, Inc., version=8, architecture=x86_64, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, name=centos-stream, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-02T15:12:53.453 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:53 smithi042 podman[57498]: 2024-04-02 15:12:53.360656159 +0000 UTC m=+0.097181201 container create db888e4baabe94977c69bef666e7afadccb32b770718e9099ae36a12229eb27a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-1-deactivate, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, name=centos-stream, release=754, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-02T15:12:53.453 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:53 smithi042 podman[57498]: 2024-04-02 15:12:53.283162055 +0000 UTC m=+0.019687114 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:12:53.790 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:12:53.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:12:53.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:53.532Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:12:53.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:12:53 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:12:53.532Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:12:53.791 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:53 smithi042 podman[57498]: 2024-04-02 15:12:53.571706581 +0000 UTC m=+0.308231628 container init db888e4baabe94977c69bef666e7afadccb32b770718e9099ae36a12229eb27a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-1-deactivate, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, release=754, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., RELEASE=HEAD, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, architecture=x86_64, GIT_CLEAN=True, name=centos-stream, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T15:12:53.791 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:53 smithi042 podman[57498]: 2024-04-02 15:12:53.574909791 +0000 UTC m=+0.311434833 container start db888e4baabe94977c69bef666e7afadccb32b770718e9099ae36a12229eb27a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-1-deactivate, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, maintainer=Guillaume Abrioux , distribution-scope=public, version=8, vcs-type=git, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870) 2024-04-02T15:12:53.791 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:53 smithi042 podman[57498]: 2024-04-02 15:12:53.638125256 +0000 UTC m=+0.374650327 container attach db888e4baabe94977c69bef666e7afadccb32b770718e9099ae36a12229eb27a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-1-deactivate, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, vendor=Red Hat, Inc., GIT_CLEAN=True, name=centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, release=754, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, version=8, vcs-type=git) 2024-04-02T15:12:54.201 INFO:journalctl@ceph.osd.1.smithi042.stdout:Apr 02 15:12:53 smithi042 podman[57498]: 2024-04-02 15:12:53.869690114 +0000 UTC m=+0.606215175 container died db888e4baabe94977c69bef666e7afadccb32b770718e9099ae36a12229eb27a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-1-deactivate, GIT_BRANCH=HEAD, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, distribution-scope=public, name=centos-stream, release=754, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, version=8, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T15:12:54.553 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.1.service' 2024-04-02T15:12:54.599 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:12:54.600 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-02T15:12:54.600 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-02T15:12:54.600 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.2 2024-04-02T15:12:54.790 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:12:54 smithi042 systemd[1]: Stopping Ceph osd.2 for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:12:54.790 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:12:54 smithi042 bash[57576]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.2" found: no such container 2024-04-02T15:12:55.290 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:12:54 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-2[38215]: 2024-04-02T15:12:54.809+0000 7f64a4717700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:12:55.290 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:12:54 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-2[38215]: 2024-04-02T15:12:54.809+0000 7f64a4717700 -1 osd.2 97 *** Got signal Terminated *** 2024-04-02T15:12:55.290 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:12:54 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-2[38215]: 2024-04-02T15:12:54.809+0000 7f64a4717700 -1 osd.2 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:13:00.607 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:13:00 smithi042 podman[57587]: 2024-04-02 15:13:00.348760153 +0000 UTC m=+5.655045562 container died f2d6ce6b9bfa611aff3ba240755af1f7ccab73d326b4b9b4c90a781b550a4503 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-2, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, architecture=x86_64, GIT_CLEAN=True, name=centos-stream, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.component=centos-stream-container, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, version=8, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.openshift.tags=base centos centos-stream, release=754) 2024-04-02T15:13:00.879 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:13:00 smithi042 podman[57587]: 2024-04-02 15:13:00.686047507 +0000 UTC m=+5.992332904 container cleanup f2d6ce6b9bfa611aff3ba240755af1f7ccab73d326b4b9b4c90a781b550a4503 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-2, io.buildah.version=1.19.8, release=754, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, name=centos-stream, ceph=True, distribution-scope=public, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., version=8, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream) 2024-04-02T15:13:00.879 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:13:00 smithi042 bash[57587]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-2 2024-04-02T15:13:00.879 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:13:00 smithi042 podman[57605]: 2024-04-02 15:13:00.862431433 +0000 UTC m=+0.513318612 container remove f2d6ce6b9bfa611aff3ba240755af1f7ccab73d326b4b9b4c90a781b550a4503 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-2, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, maintainer=Guillaume Abrioux , version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True) 2024-04-02T15:13:00.880 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:13:00 smithi042 bash[57617]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.2" found: no such container 2024-04-02T15:13:01.256 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:13:01 smithi042 podman[57684]: 2024-04-02 15:13:01.033167767 +0000 UTC m=+0.019374192 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:13:01.257 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:13:01 smithi042 podman[57684]: 2024-04-02 15:13:01.13820141 +0000 UTC m=+0.124407815 container create ed296b86a9192ffd3e673b00b6eee8c90a917bca86326182271c94b46d7aa6ca (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-2-deactivate, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, GIT_BRANCH=HEAD, distribution-scope=public, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, release=754, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., RELEASE=HEAD, io.openshift.expose-services=, version=8, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-02T15:13:01.540 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:13:01 smithi042 podman[57684]: 2024-04-02 15:13:01.373990247 +0000 UTC m=+0.360196655 container init ed296b86a9192ffd3e673b00b6eee8c90a917bca86326182271c94b46d7aa6ca (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-2-deactivate, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, RELEASE=HEAD, name=centos-stream, build-date=2022-05-03T08:36:31.336870, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.expose-services=, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-02T15:13:01.540 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:13:01 smithi042 podman[57684]: 2024-04-02 15:13:01.377203253 +0000 UTC m=+0.363409673 container start ed296b86a9192ffd3e673b00b6eee8c90a917bca86326182271c94b46d7aa6ca (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-2-deactivate, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vendor=Red Hat, Inc., vcs-type=git, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.openshift.expose-services=, distribution-scope=public, ceph=True, com.redhat.component=centos-stream-container, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True) 2024-04-02T15:13:01.540 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:13:01 smithi042 podman[57684]: 2024-04-02 15:13:01.448946106 +0000 UTC m=+0.435152534 container attach ed296b86a9192ffd3e673b00b6eee8c90a917bca86326182271c94b46d7aa6ca (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-2-deactivate, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, version=8, name=centos-stream, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, vcs-type=git, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD) 2024-04-02T15:13:01.954 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:13:01 smithi042 podman[57684]: 2024-04-02 15:13:01.676472666 +0000 UTC m=+0.662679079 container died ed296b86a9192ffd3e673b00b6eee8c90a917bca86326182271c94b46d7aa6ca (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-2-deactivate, release=754, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, name=centos-stream, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, vcs-type=git, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-02T15:13:01.954 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:13:01.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:13:01.955 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:01.645Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:13:01.955 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:01 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:01.645Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:13:02.220 INFO:journalctl@ceph.osd.2.smithi042.stdout:Apr 02 15:13:02 smithi042 podman[57710]: 2024-04-02 15:13:02.210599583 +0000 UTC m=+0.528799105 container remove ed296b86a9192ffd3e673b00b6eee8c90a917bca86326182271c94b46d7aa6ca (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vendor=Red Hat, Inc., distribution-scope=public, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.openshift.expose-services=, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream) 2024-04-02T15:13:02.225 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.2.service' 2024-04-02T15:13:02.265 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:13:02.266 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-02T15:13:02.266 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-02T15:13:02.266 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.3 2024-04-02T15:13:02.542 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:02 smithi042 systemd[1]: Stopping Ceph osd.3 for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:13:02.542 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:02 smithi042 bash[57763]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.3" found: no such container 2024-04-02T15:13:02.542 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:02 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-3[41200]: 2024-04-02T15:13:02.478+0000 7fde91da8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:13:02.542 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:02 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-3[41200]: 2024-04-02T15:13:02.478+0000 7fde91da8700 -1 osd.3 97 *** Got signal Terminated *** 2024-04-02T15:13:02.543 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:02 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-3[41200]: 2024-04-02T15:13:02.478+0000 7fde91da8700 -1 osd.3 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:13:03.790 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:13:03.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:13:03.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:03.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:13:03.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:03 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:03.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:13:06.474 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:06 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T15:13:06.062+0000 7f8a47422700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:42.579967+0000 front 2024-04-02T15:12:42.579986+0000 (oldest deadline 2024-04-02T15:13:05.479846+0000) 2024-04-02T15:13:07.474 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:07 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T15:13:07.066+0000 7f8a47422700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:42.579967+0000 front 2024-04-02T15:12:42.579986+0000 (oldest deadline 2024-04-02T15:13:05.479846+0000) 2024-04-02T15:13:07.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:07 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:07.252+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:08.250 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:07 smithi042 podman[57776]: 2024-04-02 15:13:07.952023944 +0000 UTC m=+5.559177866 container died c5d80c30db4c2cf6fac46194e1881eee68889ca89907824f917b27e1e50b2da8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-3, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, vendor=Red Hat, Inc., ceph=True, io.k8s.display-name=CentOS Stream 8, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, RELEASE=HEAD, release=754, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-02T15:13:08.438 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:08 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T15:13:08.022+0000 7f8a47422700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:42.579967+0000 front 2024-04-02T15:12:42.579986+0000 (oldest deadline 2024-04-02T15:13:05.479846+0000) 2024-04-02T15:13:08.439 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:08 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:08.248+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:08.540 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:08 smithi042 podman[57776]: 2024-04-02 15:13:08.338245085 +0000 UTC m=+5.945399007 container cleanup c5d80c30db4c2cf6fac46194e1881eee68889ca89907824f917b27e1e50b2da8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-3, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, architecture=x86_64, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T15:13:08.540 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:08 smithi042 bash[57776]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-3 2024-04-02T15:13:08.541 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:08 smithi042 bash[57805]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.3" found: no such container 2024-04-02T15:13:08.724 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:08 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:08.452+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:44.045744+0000 front 2024-04-02T15:12:44.045755+0000 (oldest deadline 2024-04-02T15:13:08.145703+0000) 2024-04-02T15:13:08.925 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:08 smithi042 podman[57793]: 2024-04-02 15:13:08.598131529 +0000 UTC m=+0.644341926 container remove c5d80c30db4c2cf6fac46194e1881eee68889ca89907824f917b27e1e50b2da8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-3, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, architecture=x86_64, vcs-type=git, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, maintainer=Guillaume Abrioux , ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, name=centos-stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, GIT_CLEAN=True) 2024-04-02T15:13:08.925 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:08 smithi042 podman[57868]: 2024-04-02 15:13:08.738929804 +0000 UTC m=+0.019111437 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:13:08.925 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:08 smithi042 podman[57868]: 2024-04-02 15:13:08.840109195 +0000 UTC m=+0.120290852 container create 27e497fa3cdf430bb89680283236f44677f7e4f11050cf9d1aa6bbe1f4cc25af (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-3-deactivate, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., distribution-scope=public, architecture=x86_64, name=centos-stream, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, version=8) 2024-04-02T15:13:09.224 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:08 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T15:13:08.974+0000 7f8a47422700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:42.579967+0000 front 2024-04-02T15:12:42.579986+0000 (oldest deadline 2024-04-02T15:13:05.479846+0000) 2024-04-02T15:13:09.290 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:09 smithi042 podman[57868]: 2024-04-02 15:13:09.025373967 +0000 UTC m=+0.305555601 container init 27e497fa3cdf430bb89680283236f44677f7e4f11050cf9d1aa6bbe1f4cc25af (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-3-deactivate, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.openshift.expose-services=, maintainer=Guillaume Abrioux , ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, GIT_CLEAN=True, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, vcs-type=git, version=8, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T15:13:09.291 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:09 smithi042 podman[57868]: 2024-04-02 15:13:09.028448813 +0000 UTC m=+0.308630434 container start 27e497fa3cdf430bb89680283236f44677f7e4f11050cf9d1aa6bbe1f4cc25af (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-3-deactivate, io.buildah.version=1.19.8, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, release=754, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, RELEASE=HEAD, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.openshift.expose-services=, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, version=8, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T15:13:09.291 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:09 smithi042 podman[57868]: 2024-04-02 15:13:09.117188183 +0000 UTC m=+0.397369810 container attach 27e497fa3cdf430bb89680283236f44677f7e4f11050cf9d1aa6bbe1f4cc25af (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-3-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, ceph=True, RELEASE=HEAD, architecture=x86_64, vcs-type=git, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream) 2024-04-02T15:13:09.572 INFO:journalctl@ceph.osd.3.smithi042.stdout:Apr 02 15:13:09 smithi042 podman[57868]: 2024-04-02 15:13:09.313081172 +0000 UTC m=+0.593262790 container died 27e497fa3cdf430bb89680283236f44677f7e4f11050cf9d1aa6bbe1f4cc25af (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-3-deactivate, version=8, vendor=Red Hat, Inc., vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, name=centos-stream, GIT_BRANCH=HEAD, release=754, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, distribution-scope=public, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, maintainer=Guillaume Abrioux ) 2024-04-02T15:13:09.725 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:09 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:09.491+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:44.045744+0000 front 2024-04-02T15:12:44.045755+0000 (oldest deadline 2024-04-02T15:13:08.145703+0000) 2024-04-02T15:13:09.725 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:09 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:09.274+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:09.844 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.3.service' 2024-04-02T15:13:09.885 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:13:09.885 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-02T15:13:09.886 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-02T15:13:09.886 DEBUG:teuthology.orchestra.run.smithi169:> sudo systemctl stop ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.4 2024-04-02T15:13:10.224 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:09 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:09.974+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:10.225 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:09 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T15:13:09.928+0000 7f8a47422700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:42.579967+0000 front 2024-04-02T15:12:42.579986+0000 (oldest deadline 2024-04-02T15:13:05.479846+0000) 2024-04-02T15:13:10.225 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:09 smithi169 systemd[1]: Stopping Ceph osd.4 for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:13:10.225 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:09 smithi169 bash[47128]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.4" found: no such container 2024-04-02T15:13:10.225 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:10 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T15:13:10.096+0000 7f8a4a286700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:13:10.225 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:10 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T15:13:10.096+0000 7f8a4a286700 -1 osd.4 97 *** Got signal Terminated *** 2024-04-02T15:13:10.226 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:10 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T15:13:10.096+0000 7f8a4a286700 -1 osd.4 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:13:10.521 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:10 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:10.239+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:10.964 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:10 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:10.534+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:44.045744+0000 front 2024-04-02T15:12:44.045755+0000 (oldest deadline 2024-04-02T15:13:08.145703+0000) 2024-04-02T15:13:11.224 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:10 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T15:13:10.977+0000 7f8a47422700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:42.579967+0000 front 2024-04-02T15:12:42.579986+0000 (oldest deadline 2024-04-02T15:13:05.479846+0000) 2024-04-02T15:13:11.225 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:11 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:11.014+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:11.548 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:11 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:11.260+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:11.974 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:11 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:11.562+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:44.045744+0000 front 2024-04-02T15:12:44.045755+0000 (oldest deadline 2024-04-02T15:13:08.145703+0000) 2024-04-02T15:13:12.040 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:13:11.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:13:12.040 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:11.645Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:13:12.040 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:11 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:11.646Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:13:12.261 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:12 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:12.059+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:12.262 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:12 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T15:13:12.008+0000 7f8a47422700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:42.579967+0000 front 2024-04-02T15:12:42.579986+0000 (oldest deadline 2024-04-02T15:13:05.479846+0000) 2024-04-02T15:13:12.582 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:12 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:12.275+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:12.974 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:12 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:12.596+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:44.045744+0000 front 2024-04-02T15:12:44.045755+0000 (oldest deadline 2024-04-02T15:13:08.145703+0000) 2024-04-02T15:13:13.247 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:13 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:13.069+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:13.247 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:12 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T15:13:12.992+0000 7f8a47422700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:42.579967+0000 front 2024-04-02T15:12:42.579986+0000 (oldest deadline 2024-04-02T15:13:05.479846+0000) 2024-04-02T15:13:13.598 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:13 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:13.260+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:13.790 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:13:13.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:13:13.790 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:13.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:13:13.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:13 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:13.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:13:13.974 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:13 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:13.611+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:44.045744+0000 front 2024-04-02T15:12:44.045755+0000 (oldest deadline 2024-04-02T15:13:08.145703+0000) 2024-04-02T15:13:13.975 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:13 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:13.611+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:48.146236+0000 front 2024-04-02T15:12:48.146225+0000 (oldest deadline 2024-04-02T15:13:13.446202+0000) 2024-04-02T15:13:14.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:14 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:14.233+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:14.475 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:14 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T15:13:14.031+0000 7f8a47422700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:42.579967+0000 front 2024-04-02T15:12:42.579986+0000 (oldest deadline 2024-04-02T15:13:05.479846+0000) 2024-04-02T15:13:14.476 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:14 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:14.117+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:14.476 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:14 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:14.117+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:14.974 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:14 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:14.594+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:44.045744+0000 front 2024-04-02T15:12:44.045755+0000 (oldest deadline 2024-04-02T15:13:08.145703+0000) 2024-04-02T15:13:14.975 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:14 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:14.594+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:48.146236+0000 front 2024-04-02T15:12:48.146225+0000 (oldest deadline 2024-04-02T15:13:13.446202+0000) 2024-04-02T15:13:15.439 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:15.240+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:15.440 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T15:13:15.000+0000 7f8a47422700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:42.579967+0000 front 2024-04-02T15:12:42.579986+0000 (oldest deadline 2024-04-02T15:13:05.479846+0000) 2024-04-02T15:13:15.440 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4[29186]: 2024-04-02T15:13:15.000+0000 7f8a47422700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:47.180822+0000 front 2024-04-02T15:12:47.180700+0000 (oldest deadline 2024-04-02T15:13:14.781390+0000) 2024-04-02T15:13:15.440 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:15.107+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:15.440 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:15.107+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:15.724 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:15 smithi169 podman[47141]: 2024-04-02 15:13:15.45301953 +0000 UTC m=+5.466411698 container died efc569199c212208c8a2f8d66831937238e86c74657dd1ca6a6c2f266114349a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4, maintainer=Guillaume Abrioux , GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, ceph=True, vendor=Red Hat, Inc., vcs-type=git, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, name=centos-stream) 2024-04-02T15:13:15.725 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:15.549+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:44.045744+0000 front 2024-04-02T15:12:44.045755+0000 (oldest deadline 2024-04-02T15:13:08.145703+0000) 2024-04-02T15:13:15.725 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:15 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:15.549+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:48.146236+0000 front 2024-04-02T15:12:48.146225+0000 (oldest deadline 2024-04-02T15:13:13.446202+0000) 2024-04-02T15:13:16.069 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:15 smithi169 podman[47141]: 2024-04-02 15:13:15.824137721 +0000 UTC m=+5.837529886 container cleanup efc569199c212208c8a2f8d66831937238e86c74657dd1ca6a6c2f266114349a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, architecture=x86_64, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, vcs-type=git, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, maintainer=Guillaume Abrioux , name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T15:13:16.070 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:15 smithi169 bash[47141]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4 2024-04-02T15:13:16.070 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:15 smithi169 bash[47172]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.4" found: no such container 2024-04-02T15:13:16.360 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:16.256+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:16.360 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:16.136+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:16.361 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:16.136+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:16.361 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:16 smithi169 podman[47158]: 2024-04-02 15:13:16.08372609 +0000 UTC m=+0.622557953 container remove efc569199c212208c8a2f8d66831937238e86c74657dd1ca6a6c2f266114349a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, com.redhat.component=centos-stream-container, ceph=True, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.openshift.expose-services=, distribution-scope=public, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, release=754) 2024-04-02T15:13:16.361 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:16 smithi169 podman[47236]: 2024-04-02 15:13:16.166779459 +0000 UTC m=+0.019090207 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:13:16.361 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:16 smithi169 podman[47236]: 2024-04-02 15:13:16.27643781 +0000 UTC m=+0.128748581 container create cd71ef4eccac2a0f89f60afc921fc500ee914cff97e3f8ad3d4a5767fd32ec76 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4-deactivate, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, architecture=x86_64, release=754, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, ceph=True, vcs-type=git) 2024-04-02T15:13:16.724 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:16.592+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:44.045744+0000 front 2024-04-02T15:12:44.045755+0000 (oldest deadline 2024-04-02T15:13:08.145703+0000) 2024-04-02T15:13:16.725 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:16 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:16.592+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:48.146236+0000 front 2024-04-02T15:12:48.146225+0000 (oldest deadline 2024-04-02T15:13:13.446202+0000) 2024-04-02T15:13:16.725 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:16 smithi169 podman[47236]: 2024-04-02 15:13:16.495570641 +0000 UTC m=+0.347881369 container init cd71ef4eccac2a0f89f60afc921fc500ee914cff97e3f8ad3d4a5767fd32ec76 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4-deactivate, GIT_CLEAN=True, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., RELEASE=HEAD, ceph=True, version=8, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, maintainer=Guillaume Abrioux , distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container) 2024-04-02T15:13:16.725 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:16 smithi169 podman[47236]: 2024-04-02 15:13:16.498735863 +0000 UTC m=+0.351046595 container start cd71ef4eccac2a0f89f60afc921fc500ee914cff97e3f8ad3d4a5767fd32ec76 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4-deactivate, ceph=True, name=centos-stream, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, GIT_CLEAN=True, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , version=8, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, vendor=Red Hat, Inc.) 2024-04-02T15:13:16.725 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:16 smithi169 podman[47236]: 2024-04-02 15:13:16.570428023 +0000 UTC m=+0.422738753 container attach cd71ef4eccac2a0f89f60afc921fc500ee914cff97e3f8ad3d4a5767fd32ec76 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4-deactivate, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_CLEAN=True, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, release=754, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, maintainer=Guillaume Abrioux , version=8, vcs-type=git, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-02T15:13:17.085 INFO:journalctl@ceph.osd.4.smithi169.stdout:Apr 02 15:13:16 smithi169 podman[47236]: 2024-04-02 15:13:16.785078904 +0000 UTC m=+0.637389628 container died cd71ef4eccac2a0f89f60afc921fc500ee914cff97e3f8ad3d4a5767fd32ec76 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-4-deactivate, version=8, vcs-type=git, io.buildah.version=1.19.8, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=HEAD, distribution-scope=public, name=centos-stream, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0) 2024-04-02T15:13:17.351 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:17.148+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:17.352 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:17.148+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:17.352 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:17.235+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:17.352 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:17.235+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.002181+0000 front 2024-04-02T15:12:51.002193+0000 (oldest deadline 2024-04-02T15:13:16.902150+0000) 2024-04-02T15:13:17.372 DEBUG:teuthology.orchestra.run.smithi169:> sudo pkill -f 'journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.4.service' 2024-04-02T15:13:17.414 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:13:17.414 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-02T15:13:17.415 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-02T15:13:17.415 DEBUG:teuthology.orchestra.run.smithi169:> sudo systemctl stop ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.5 2024-04-02T15:13:17.622 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:17 smithi169 systemd[1]: Stopping Ceph osd.5 for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:13:17.622 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:17 smithi169 bash[47319]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.5" found: no such container 2024-04-02T15:13:17.622 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:17.606+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:44.045744+0000 front 2024-04-02T15:12:44.045755+0000 (oldest deadline 2024-04-02T15:13:08.145703+0000) 2024-04-02T15:13:17.622 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:17.606+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:48.146236+0000 front 2024-04-02T15:12:48.146225+0000 (oldest deadline 2024-04-02T15:13:13.446202+0000) 2024-04-02T15:13:17.623 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:17.635+0000 7fa0c9c67700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:13:17.623 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:17.635+0000 7fa0c9c67700 -1 osd.5 97 *** Got signal Terminated *** 2024-04-02T15:13:17.623 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:17 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:17.635+0000 7fa0c9c67700 -1 osd.5 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:13:18.474 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:18.209+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:18.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:18.209+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.002181+0000 front 2024-04-02T15:12:51.002193+0000 (oldest deadline 2024-04-02T15:13:16.902150+0000) 2024-04-02T15:13:18.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:18.175+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:18.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:18.175+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:18.974 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:18.645+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:44.045744+0000 front 2024-04-02T15:12:44.045755+0000 (oldest deadline 2024-04-02T15:13:08.145703+0000) 2024-04-02T15:13:18.975 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:18 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:18.645+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:48.146236+0000 front 2024-04-02T15:12:48.146225+0000 (oldest deadline 2024-04-02T15:13:13.446202+0000) 2024-04-02T15:13:19.474 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:19.258+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:19.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:19.258+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.002181+0000 front 2024-04-02T15:12:51.002193+0000 (oldest deadline 2024-04-02T15:13:16.902150+0000) 2024-04-02T15:13:19.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:19.164+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:19.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:19.164+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:19.974 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:19.655+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:44.045744+0000 front 2024-04-02T15:12:44.045755+0000 (oldest deadline 2024-04-02T15:13:08.145703+0000) 2024-04-02T15:13:19.974 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:19 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:19.655+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:48.146236+0000 front 2024-04-02T15:12:48.146225+0000 (oldest deadline 2024-04-02T15:13:13.446202+0000) 2024-04-02T15:13:20.474 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:20 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:20.184+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:20.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:20 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:20.184+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:20.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:20 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:20.238+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:20.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:20 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:20.238+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.002181+0000 front 2024-04-02T15:12:51.002193+0000 (oldest deadline 2024-04-02T15:13:16.902150+0000) 2024-04-02T15:13:20.974 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:20 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:20.697+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:44.045744+0000 front 2024-04-02T15:12:44.045755+0000 (oldest deadline 2024-04-02T15:13:08.145703+0000) 2024-04-02T15:13:20.978 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:20 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:20.697+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:48.146236+0000 front 2024-04-02T15:12:48.146225+0000 (oldest deadline 2024-04-02T15:13:13.446202+0000) 2024-04-02T15:13:21.474 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:21 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:21.221+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:21.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:21 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:21.221+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.002181+0000 front 2024-04-02T15:12:51.002193+0000 (oldest deadline 2024-04-02T15:13:16.902150+0000) 2024-04-02T15:13:21.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:21 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:21.136+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:21.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:21 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:21.136+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:21.899 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:13:21.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:13:21.900 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:21.647Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:13:21.900 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:21 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:21.647Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:13:21.974 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:21 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:21.707+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:44.045744+0000 front 2024-04-02T15:12:44.045755+0000 (oldest deadline 2024-04-02T15:13:08.145703+0000) 2024-04-02T15:13:21.974 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:21 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5[32128]: 2024-04-02T15:13:21.707+0000 7fa0c5600700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:48.146236+0000 front 2024-04-02T15:12:48.146225+0000 (oldest deadline 2024-04-02T15:13:13.446202+0000) 2024-04-02T15:13:22.474 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:22 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:22.142+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:22.474 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:22 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:22.142+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:22.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:22 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:22.142+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:22.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:22 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:22.269+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:22.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:22 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:22.269+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.002181+0000 front 2024-04-02T15:12:51.002193+0000 (oldest deadline 2024-04-02T15:13:16.902150+0000) 2024-04-02T15:13:22.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:22 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:22.269+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:56.902504+0000 front 2024-04-02T15:12:56.902534+0000 (oldest deadline 2024-04-02T15:13:21.602526+0000) 2024-04-02T15:13:23.406 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:23 smithi169 podman[47330]: 2024-04-02 15:13:23.106078534 +0000 UTC m=+5.566015671 container died d543be94df6d0b0b3172fdb5e3e6c368b0201e516ef5dc070aec872920024b7a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, architecture=x86_64, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, ceph=True, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.openshift.expose-services=, vendor=Red Hat, Inc., version=8) 2024-04-02T15:13:23.406 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:23 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:23.303+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:23.406 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:23 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:23.303+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.002181+0000 front 2024-04-02T15:12:51.002193+0000 (oldest deadline 2024-04-02T15:13:16.902150+0000) 2024-04-02T15:13:23.406 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:23 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:23.303+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:56.902504+0000 front 2024-04-02T15:12:56.902534+0000 (oldest deadline 2024-04-02T15:13:21.602526+0000) 2024-04-02T15:13:23.407 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:23 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:23.127+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:23.407 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:23 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:23.128+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:23.407 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:23 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:23.128+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:23.673 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:23 smithi169 podman[47330]: 2024-04-02 15:13:23.494086088 +0000 UTC m=+5.954023219 container cleanup d543be94df6d0b0b3172fdb5e3e6c368b0201e516ef5dc070aec872920024b7a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, distribution-scope=public, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754) 2024-04-02T15:13:23.673 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:23 smithi169 bash[47330]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5 2024-04-02T15:13:23.673 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:23 smithi169 bash[47359]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.5" found: no such container 2024-04-02T15:13:23.790 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:13:23.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:13:23.790 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:23.534Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:13:23.790 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:23 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:23.534Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:13:23.974 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:23 smithi169 podman[47347]: 2024-04-02 15:13:23.687576813 +0000 UTC m=+0.577897813 container remove d543be94df6d0b0b3172fdb5e3e6c368b0201e516ef5dc070aec872920024b7a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5, name=centos-stream, vcs-type=git, release=754, io.openshift.expose-services=, architecture=x86_64, version=8, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8) 2024-04-02T15:13:23.975 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:23 smithi169 podman[47422]: 2024-04-02 15:13:23.834091019 +0000 UTC m=+0.018471073 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:13:23.975 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:23 smithi169 podman[47422]: 2024-04-02 15:13:23.931179886 +0000 UTC m=+0.115559920 container create ee0f3fa99ec7cf680bd6a68c750c5491d4022fc4d01109fcf2eace9f59b40f4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5-deactivate, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, ceph=True, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, distribution-scope=public, architecture=x86_64, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, RELEASE=HEAD, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T15:13:24.286 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:24 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:24.165+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:24.286 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:24 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:24.165+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:24.286 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:24 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:24.165+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:24.287 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:24 smithi169 podman[47422]: 2024-04-02 15:13:24.150170097 +0000 UTC m=+0.334550146 container init ee0f3fa99ec7cf680bd6a68c750c5491d4022fc4d01109fcf2eace9f59b40f4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5-deactivate, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.buildah.version=1.19.8, architecture=x86_64, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, version=8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8) 2024-04-02T15:13:24.287 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:24 smithi169 podman[47422]: 2024-04-02 15:13:24.153421777 +0000 UTC m=+0.337801812 container start ee0f3fa99ec7cf680bd6a68c750c5491d4022fc4d01109fcf2eace9f59b40f4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5-deactivate, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, vcs-type=git, io.buildah.version=1.19.8) 2024-04-02T15:13:24.287 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:24 smithi169 podman[47422]: 2024-04-02 15:13:24.250268077 +0000 UTC m=+0.434648111 container attach ee0f3fa99ec7cf680bd6a68c750c5491d4022fc4d01109fcf2eace9f59b40f4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5-deactivate, vendor=Red Hat, Inc., vcs-type=git, io.buildah.version=1.19.8, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, version=8, build-date=2022-05-03T08:36:31.336870, name=centos-stream, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-02T15:13:24.724 INFO:journalctl@ceph.osd.5.smithi169.stdout:Apr 02 15:13:24 smithi169 podman[47422]: 2024-04-02 15:13:24.433104242 +0000 UTC m=+0.617484290 container died ee0f3fa99ec7cf680bd6a68c750c5491d4022fc4d01109fcf2eace9f59b40f4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-5-deactivate, com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, ceph=True, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, version=8, GIT_CLEAN=True, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, distribution-scope=public, vcs-type=git, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754) 2024-04-02T15:13:24.725 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:24 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:24.299+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:24.725 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:24 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:24.299+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.002181+0000 front 2024-04-02T15:12:51.002193+0000 (oldest deadline 2024-04-02T15:13:16.902150+0000) 2024-04-02T15:13:24.725 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:24 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:24.299+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:56.902504+0000 front 2024-04-02T15:12:56.902534+0000 (oldest deadline 2024-04-02T15:13:21.602526+0000) 2024-04-02T15:13:25.012 DEBUG:teuthology.orchestra.run.smithi169:> sudo pkill -f 'journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.5.service' 2024-04-02T15:13:25.056 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:13:25.056 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-02T15:13:25.057 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-02T15:13:25.057 DEBUG:teuthology.orchestra.run.smithi169:> sudo systemctl stop ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.6 2024-04-02T15:13:25.287 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:25 smithi169 systemd[1]: Stopping Ceph osd.6 for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:13:25.287 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:25 smithi169 bash[47501]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.6" found: no such container 2024-04-02T15:13:25.288 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:25 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:25.150+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:25.288 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:25 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:25.150+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:25.288 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:25 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:25.150+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:25.724 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:25 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:25.301+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:25.725 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:25 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:25.301+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.002181+0000 front 2024-04-02T15:12:51.002193+0000 (oldest deadline 2024-04-02T15:13:16.902150+0000) 2024-04-02T15:13:25.725 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:25 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:25.301+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:56.902504+0000 front 2024-04-02T15:12:56.902534+0000 (oldest deadline 2024-04-02T15:13:21.602526+0000) 2024-04-02T15:13:25.725 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:25 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:25.324+0000 7ff6d24fa700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:13:25.726 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:25 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:25.324+0000 7ff6d24fa700 -1 osd.6 97 *** Got signal Terminated *** 2024-04-02T15:13:25.726 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:25 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:25.324+0000 7ff6d24fa700 -1 osd.6 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:13:26.474 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:26 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:26.351+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:26.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:26 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:26.351+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.002181+0000 front 2024-04-02T15:12:51.002193+0000 (oldest deadline 2024-04-02T15:13:16.902150+0000) 2024-04-02T15:13:26.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:26 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:26.351+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:56.902504+0000 front 2024-04-02T15:12:56.902534+0000 (oldest deadline 2024-04-02T15:13:21.602526+0000) 2024-04-02T15:13:26.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:26 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:26.128+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:26.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:26 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:26.128+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:26.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:26 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:26.128+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:27.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:27 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:27.304+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:27.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:27 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:27.304+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.002181+0000 front 2024-04-02T15:12:51.002193+0000 (oldest deadline 2024-04-02T15:13:16.902150+0000) 2024-04-02T15:13:27.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:27 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:27.304+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:56.902504+0000 front 2024-04-02T15:12:56.902534+0000 (oldest deadline 2024-04-02T15:13:21.602526+0000) 2024-04-02T15:13:27.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:27 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:27.172+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:27.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:27 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:27.172+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:27.476 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:27 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:27.172+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:28.271 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:28 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:28.200+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:28.272 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:28 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:28.200+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:28.272 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:28 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:28.200+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:28.724 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:28 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:28.285+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:28.724 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:28 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:28.285+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.002181+0000 front 2024-04-02T15:12:51.002193+0000 (oldest deadline 2024-04-02T15:13:16.902150+0000) 2024-04-02T15:13:28.725 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:28 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:28.285+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:56.902504+0000 front 2024-04-02T15:12:56.902534+0000 (oldest deadline 2024-04-02T15:13:21.602526+0000) 2024-04-02T15:13:29.474 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:29 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:29.325+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:29.474 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:29 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:29.325+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.002181+0000 front 2024-04-02T15:12:51.002193+0000 (oldest deadline 2024-04-02T15:13:16.902150+0000) 2024-04-02T15:13:29.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:29 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:29.325+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:56.902504+0000 front 2024-04-02T15:12:56.902534+0000 (oldest deadline 2024-04-02T15:13:21.602526+0000) 2024-04-02T15:13:29.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:29 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:29.325+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-04-02T15:13:03.303595+0000 front 2024-04-02T15:13:03.303508+0000 (oldest deadline 2024-04-02T15:13:28.603575+0000) 2024-04-02T15:13:29.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:29 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:29.190+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:29.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:29 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:29.190+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:29.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:29 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:29.190+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:30.474 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:30 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:30.281+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:41.001235+0000 front 2024-04-02T15:12:41.001126+0000 (oldest deadline 2024-04-02T15:13:06.901193+0000) 2024-04-02T15:13:30.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:30 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:30.281+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.002181+0000 front 2024-04-02T15:12:51.002193+0000 (oldest deadline 2024-04-02T15:13:16.902150+0000) 2024-04-02T15:13:30.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:30 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:30.281+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:56.902504+0000 front 2024-04-02T15:12:56.902534+0000 (oldest deadline 2024-04-02T15:13:21.602526+0000) 2024-04-02T15:13:30.475 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:30 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6[35028]: 2024-04-02T15:13:30.281+0000 7ff6ce694700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-04-02T15:13:03.303595+0000 front 2024-04-02T15:13:03.303508+0000 (oldest deadline 2024-04-02T15:13:28.603575+0000) 2024-04-02T15:13:30.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:30 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:30.170+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:30.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:30 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:30.170+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:30.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:30 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:30.170+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:30.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:30 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:30.170+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-04-02T15:13:07.146841+0000 front 2024-04-02T15:13:07.146830+0000 (oldest deadline 2024-04-02T15:13:29.446713+0000) 2024-04-02T15:13:30.974 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:30 smithi169 podman[47514]: 2024-04-02 15:13:30.708415821 +0000 UTC m=+5.518268399 container died 34c8a641d867df742eac25f2358cc95888b5ad4723ebc06492175d9525948b37 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, name=centos-stream, vendor=Red Hat, Inc., RELEASE=HEAD) 2024-04-02T15:13:31.311 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:31.197+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:31.312 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:31.197+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:31.312 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:31.197+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:31.312 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:31 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:31.197+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-04-02T15:13:07.146841+0000 front 2024-04-02T15:13:07.146830+0000 (oldest deadline 2024-04-02T15:13:29.446713+0000) 2024-04-02T15:13:31.312 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:31 smithi169 podman[47514]: 2024-04-02 15:13:31.099686579 +0000 UTC m=+5.909539143 container cleanup 34c8a641d867df742eac25f2358cc95888b5ad4723ebc06492175d9525948b37 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, ceph=True, release=754, io.openshift.tags=base centos centos-stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, vcs-type=git, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, version=8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T15:13:31.312 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:31 smithi169 bash[47514]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6 2024-04-02T15:13:31.313 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:31 smithi169 bash[47544]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.6" found: no such container 2024-04-02T15:13:31.612 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:31 smithi169 podman[47531]: 2024-04-02 15:13:31.325811899 +0000 UTC m=+0.611520670 container remove 34c8a641d867df742eac25f2358cc95888b5ad4723ebc06492175d9525948b37 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, ceph=True, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-type=git, vendor=Red Hat, Inc., GIT_CLEAN=True, name=centos-stream, GIT_BRANCH=HEAD, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux ) 2024-04-02T15:13:31.613 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:31 smithi169 podman[47611]: 2024-04-02 15:13:31.424272789 +0000 UTC m=+0.019307018 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:13:31.613 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:31 smithi169 podman[47611]: 2024-04-02 15:13:31.526622538 +0000 UTC m=+0.121656762 container create 2578fbac3af00dd41e5c6b7d380a61ba0b53e88dc1e5c4c14bcd6e734026ac5d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6-deactivate, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, vcs-type=git, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, distribution-scope=public, architecture=x86_64, version=8) 2024-04-02T15:13:31.975 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:31 smithi169 podman[47611]: 2024-04-02 15:13:31.746057647 +0000 UTC m=+0.341091874 container init 2578fbac3af00dd41e5c6b7d380a61ba0b53e88dc1e5c4c14bcd6e734026ac5d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6-deactivate, GIT_BRANCH=HEAD, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, version=8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, name=centos-stream, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.openshift.expose-services=, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True) 2024-04-02T15:13:31.975 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:31 smithi169 podman[47611]: 2024-04-02 15:13:31.749089295 +0000 UTC m=+0.344123519 container start 2578fbac3af00dd41e5c6b7d380a61ba0b53e88dc1e5c4c14bcd6e734026ac5d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6-deactivate, io.openshift.expose-services=, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, release=754, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., version=8, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T15:13:31.975 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:31 smithi169 podman[47611]: 2024-04-02 15:13:31.837620576 +0000 UTC m=+0.432654800 container attach 2578fbac3af00dd41e5c6b7d380a61ba0b53e88dc1e5c4c14bcd6e734026ac5d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6-deactivate, vcs-type=git, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, GIT_CLEAN=True, version=8, ceph=True, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux ) 2024-04-02T15:13:32.040 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:13:31.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:13:32.040 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:31.647Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:13:32.040 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:31 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:31.647Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:13:32.326 INFO:journalctl@ceph.osd.6.smithi169.stdout:Apr 02 15:13:32 smithi169 podman[47611]: 2024-04-02 15:13:32.04110661 +0000 UTC m=+0.636140837 container died 2578fbac3af00dd41e5c6b7d380a61ba0b53e88dc1e5c4c14bcd6e734026ac5d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-6-deactivate, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, io.openshift.expose-services=, release=754, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, architecture=x86_64, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True) 2024-04-02T15:13:32.327 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:32.200+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:32.327 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:32.200+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:32.327 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:32.200+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:32.327 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:32.200+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-04-02T15:13:07.146841+0000 front 2024-04-02T15:13:07.146830+0000 (oldest deadline 2024-04-02T15:13:29.446713+0000) 2024-04-02T15:13:32.620 DEBUG:teuthology.orchestra.run.smithi169:> sudo pkill -f 'journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.6.service' 2024-04-02T15:13:32.661 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:13:32.661 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-02T15:13:32.661 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-02T15:13:32.662 DEBUG:teuthology.orchestra.run.smithi169:> sudo systemctl stop ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.7 2024-04-02T15:13:32.872 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:32 smithi169 systemd[1]: Stopping Ceph osd.7 for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:13:32.872 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:32 smithi169 bash[47689]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.7" found: no such container 2024-04-02T15:13:32.872 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:32.885+0000 7fd1fee53700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:13:32.872 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:32.885+0000 7fd1fee53700 -1 osd.7 97 *** Got signal Terminated *** 2024-04-02T15:13:32.873 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:32 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:32.885+0000 7fd1fee53700 -1 osd.7 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:13:33.474 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:33 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:33.213+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:33.474 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:33 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:33.213+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:33.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:33 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:33.213+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:33.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:33 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:33.213+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-04-02T15:13:07.146841+0000 front 2024-04-02T15:13:07.146830+0000 (oldest deadline 2024-04-02T15:13:29.446713+0000) 2024-04-02T15:13:33.790 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:13:33.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:13:33.790 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:33.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:13:33.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:33 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:33.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:13:34.724 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:34 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:34.249+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:34.725 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:34 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:34.249+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:34.725 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:34 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:34.249+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:34.725 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:34 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:34.249+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-04-02T15:13:07.146841+0000 front 2024-04-02T15:13:07.146830+0000 (oldest deadline 2024-04-02T15:13:29.446713+0000) 2024-04-02T15:13:35.725 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:35 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:35.276+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:35.725 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:35 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:35.276+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:35.725 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:35 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:35.276+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:35.725 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:35 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:35.276+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-04-02T15:13:07.146841+0000 front 2024-04-02T15:13:07.146830+0000 (oldest deadline 2024-04-02T15:13:29.446713+0000) 2024-04-02T15:13:36.626 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:36 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:36.299+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:36.626 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:36 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:36.299+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:36.626 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:36 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:36.299+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:36.626 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:36 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:36.299+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-04-02T15:13:07.146841+0000 front 2024-04-02T15:13:07.146830+0000 (oldest deadline 2024-04-02T15:13:29.446713+0000) 2024-04-02T15:13:37.724 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:37 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:37.345+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-04-02T15:12:43.343373+0000 front 2024-04-02T15:12:43.343342+0000 (oldest deadline 2024-04-02T15:13:09.743618+0000) 2024-04-02T15:13:37.724 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:37 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:37.345+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-04-02T15:12:51.444644+0000 front 2024-04-02T15:12:51.444725+0000 (oldest deadline 2024-04-02T15:13:13.144515+0000) 2024-04-02T15:13:37.724 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:37 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:37.345+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-04-02T15:12:57.245493+0000 front 2024-04-02T15:12:57.245421+0000 (oldest deadline 2024-04-02T15:13:21.345377+0000) 2024-04-02T15:13:37.724 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:37 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7[37935]: 2024-04-02T15:13:37.345+0000 7fd1fc7f0700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-04-02T15:13:07.146841+0000 front 2024-04-02T15:13:07.146830+0000 (oldest deadline 2024-04-02T15:13:29.446713+0000) 2024-04-02T15:13:38.447 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:38 smithi169 podman[47700]: 2024-04-02 15:13:38.210579515 +0000 UTC m=+5.423437700 container died 817b439db33c0e0a4cca4b9b57e1e91ad51a395a67cc773b242d62b93b1c9647 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, name=centos-stream, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, vendor=Red Hat, Inc., GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, version=8, distribution-scope=public, RELEASE=HEAD, GIT_BRANCH=HEAD) 2024-04-02T15:13:38.724 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:38 smithi169 podman[47700]: 2024-04-02 15:13:38.569558282 +0000 UTC m=+5.782416482 container cleanup 817b439db33c0e0a4cca4b9b57e1e91ad51a395a67cc773b242d62b93b1c9647 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, RELEASE=HEAD, ceph=True, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, vcs-type=git, io.openshift.expose-services=, version=8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, io.k8s.display-name=CentOS Stream 8, name=centos-stream, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-02T15:13:38.724 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:38 smithi169 bash[47700]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7 2024-04-02T15:13:38.725 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:38 smithi169 bash[47744]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd.7" found: no such container 2024-04-02T15:13:39.147 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:38 smithi169 podman[47731]: 2024-04-02 15:13:38.862215847 +0000 UTC m=+0.651707831 container remove 817b439db33c0e0a4cca4b9b57e1e91ad51a395a67cc773b242d62b93b1c9647 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7, release=754, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, distribution-scope=public, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, version=8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-02T15:13:39.148 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:39 smithi169 podman[47807]: 2024-04-02 15:13:38.946809734 +0000 UTC m=+0.018571472 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:13:39.148 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:39 smithi169 podman[47807]: 2024-04-02 15:13:39.054849347 +0000 UTC m=+0.126611068 container create 72b8288f51039b048605ca388377b5e9b492387aa655568d074d466d0821a63b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7-deactivate, vcs-type=git, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.buildah.version=1.19.8, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-02T15:13:39.474 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:39 smithi169 podman[47807]: 2024-04-02 15:13:39.265607595 +0000 UTC m=+0.337369308 container init 72b8288f51039b048605ca388377b5e9b492387aa655568d074d466d0821a63b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7-deactivate, vendor=Red Hat, Inc., GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, version=8, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, distribution-scope=public) 2024-04-02T15:13:39.474 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:39 smithi169 podman[47807]: 2024-04-02 15:13:39.268644256 +0000 UTC m=+0.340405972 container start 72b8288f51039b048605ca388377b5e9b492387aa655568d074d466d0821a63b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7-deactivate, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.buildah.version=1.19.8, vcs-type=git, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vendor=Red Hat, Inc., name=centos-stream, GIT_CLEAN=True, release=754, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, RELEASE=HEAD, io.openshift.tags=base centos centos-stream) 2024-04-02T15:13:39.475 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:39 smithi169 podman[47807]: 2024-04-02 15:13:39.348783542 +0000 UTC m=+0.420545262 container attach 72b8288f51039b048605ca388377b5e9b492387aa655568d074d466d0821a63b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7-deactivate, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8) 2024-04-02T15:13:39.837 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:39 smithi169 podman[47807]: 2024-04-02 15:13:39.551557427 +0000 UTC m=+0.623319151 container died 72b8288f51039b048605ca388377b5e9b492387aa655568d074d466d0821a63b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7-deactivate, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, vcs-type=git, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vendor=Red Hat, Inc., name=centos-stream, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-02T15:13:40.101 INFO:journalctl@ceph.osd.7.smithi169.stdout:Apr 02 15:13:40 smithi169 podman[47833]: 2024-04-02 15:13:40.094062008 +0000 UTC m=+0.532862183 container remove 72b8288f51039b048605ca388377b5e9b492387aa655568d074d466d0821a63b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-osd-7-deactivate, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, ceph=True, release=754, vcs-type=git, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, name=centos-stream, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-02T15:13:40.107 DEBUG:teuthology.orchestra.run.smithi169:> sudo pkill -f 'journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@osd.7.service' 2024-04-02T15:13:40.151 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:13:40.151 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-02T15:13:40.151 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-02T15:13:40.151 DEBUG:teuthology.orchestra.run.smithi169:> sudo systemctl stop ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@prometheus.a 2024-04-02T15:13:40.382 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 systemd[1]: Stopping Ceph prometheus.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:13:40.409 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 bash[47885]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T15:13:40.667 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:13:40.397Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-02T15:13:40.667 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:13:40.397Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-02T15:13:40.667 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:13:40.397Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-02T15:13:40.667 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:13:40.397Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-02T15:13:40.667 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:13:40.397Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-02T15:13:40.667 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:13:40.397Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-02T15:13:40.668 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:13:40.397Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-02T15:13:40.668 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:13:40.397Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-02T15:13:40.668 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:13:40.397Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-02T15:13:40.668 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:13:40.397Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-02T15:13:40.668 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:13:40.397Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-02T15:13:40.668 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a[42716]: ts=2024-04-02T15:13:40.397Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-02T15:13:40.668 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 podman[47896]: 2024-04-02 15:13:40.407545755 +0000 UTC m=+0.120342272 container died 80b1a5d12473b3ca01a442b3365bd5dbbbacb1b0b313b1fb8a7a9d4e1924c230 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:13:40.974 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 podman[47896]: 2024-04-02 15:13:40.780009371 +0000 UTC m=+0.492805899 container cleanup 80b1a5d12473b3ca01a442b3365bd5dbbbacb1b0b313b1fb8a7a9d4e1924c230 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:13:40.974 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 bash[47896]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus-a 2024-04-02T15:13:40.974 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Apr 02 15:13:40 smithi169 bash[47924]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T15:13:41.038 DEBUG:teuthology.orchestra.run.smithi169:> sudo pkill -f 'journalctl -f -n 0 -u ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@prometheus.a.service' 2024-04-02T15:13:41.074 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:13:41.075 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-02T15:13:41.075 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 --force --keep-logs 2024-04-02T15:13:41.790 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:13:41.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:13:41.790 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:41.648Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:13:41.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:41 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:41.649Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:13:42.818 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:13:42 smithi042 systemd[1]: Stopping Ceph node-exporter.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:13:42.818 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:13:42 smithi042 bash[58241]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter.a" found: no such container 2024-04-02T15:13:43.091 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:13:42 smithi042 podman[58254]: 2024-04-02 15:13:42.832739504 +0000 UTC m=+0.227530531 container died d0a5ca877a2af0c19c65cf92f60a3702b55dc128d90867b307dc800a04088489 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T15:13:43.442 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:13:43 smithi042 podman[58254]: 2024-04-02 15:13:43.205321999 +0000 UTC m=+0.600113027 container cleanup d0a5ca877a2af0c19c65cf92f60a3702b55dc128d90867b307dc800a04088489 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T15:13:43.442 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:13:43 smithi042 bash[58254]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a 2024-04-02T15:13:43.442 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:13:43 smithi042 bash[58280]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter.a" found: no such container 2024-04-02T15:13:43.790 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:13:43 smithi042 podman[58268]: 2024-04-02 15:13:43.456408899 +0000 UTC m=+0.628269738 container remove d0a5ca877a2af0c19c65cf92f60a3702b55dc128d90867b307dc800a04088489 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T15:13:43.790 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:13:43 smithi042 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-02T15:13:43.791 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:13:43 smithi042 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-02T15:13:43.791 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:13:43 smithi042 systemd[1]: Stopped Ceph node-exporter.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T15:13:43.791 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Apr 02 15:13:43 smithi042 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@node-exporter.a.service: Consumed 5.210s CPU time. 2024-04-02T15:13:43.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=error ts=2024-04-02T15:13:43.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:13:43.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:43.536Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.42:8443//api/prometheus_receiver\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-04-02T15:13:43.791 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=warn ts=2024-04-02T15:13:43.536Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.169:8443/api/prometheus_receiver\": dial tcp 172.21.15.169:8443: connect: connection refused" 2024-04-02T15:13:43.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:43 smithi042 systemd[1]: Stopping Ceph alertmanager.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:13:43.792 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:43 smithi042 bash[58331]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-02T15:13:44.222 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:43 smithi042 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a[49511]: level=info ts=2024-04-02T15:13:43.945Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-02T15:13:44.223 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:43 smithi042 podman[58343]: 2024-04-02 15:13:43.965610712 +0000 UTC m=+0.285393915 container died 64e49f76da9a8efc5889cbc6cf70d6cf7eccab9993d792ef34ae0667b5b31391 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:13:44.540 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:44 smithi042 podman[58343]: 2024-04-02 15:13:44.328385567 +0000 UTC m=+0.648168821 container cleanup 64e49f76da9a8efc5889cbc6cf70d6cf7eccab9993d792ef34ae0667b5b31391 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:13:44.541 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:44 smithi042 bash[58343]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a 2024-04-02T15:13:44.541 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:44 smithi042 bash[58370]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-02T15:13:44.849 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:44 smithi042 podman[58357]: 2024-04-02 15:13:44.587614253 +0000 UTC m=+0.618708213 container remove 64e49f76da9a8efc5889cbc6cf70d6cf7eccab9993d792ef34ae0667b5b31391 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:13:44.849 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:44 smithi042 podman[58357]: 2024-04-02 15:13:44.671585402 +0000 UTC m=+0.702679333 volume remove a7855cf7da939d769415e8c0de2fbac5e8c9557ca79fcbef35918abcc8bceae0 2024-04-02T15:13:44.849 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:44 smithi042 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-02T15:13:44.849 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:44 smithi042 systemd[1]: Stopped Ceph alertmanager.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T15:13:44.850 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Apr 02 15:13:44 smithi042 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@alertmanager.a.service: Consumed 1.636s CPU time. 2024-04-02T15:14:07.392 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 --force --keep-logs 2024-04-02T15:14:09.054 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:14:08 smithi169 systemd[1]: Stopping Ceph node-exporter.b for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:14:09.054 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:14:08 smithi169 bash[48264]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter.b" found: no such container 2024-04-02T15:14:09.360 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:14:09 smithi169 podman[48277]: 2024-04-02 15:14:09.06693585 +0000 UTC m=+0.113601019 container died dbb30e649ccfc88ac848190e9f5fadeb9261894ee68a7139a310a309cbd5cedb (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T15:14:09.668 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:14:09 smithi169 podman[48277]: 2024-04-02 15:14:09.473417984 +0000 UTC m=+0.520083138 container cleanup dbb30e649ccfc88ac848190e9f5fadeb9261894ee68a7139a310a309cbd5cedb (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T15:14:09.669 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:14:09 smithi169 bash[48277]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b 2024-04-02T15:14:09.669 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:14:09 smithi169 bash[48304]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter.b" found: no such container 2024-04-02T15:14:09.975 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:14:09 smithi169 systemd[1]: Stopping Ceph grafana.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4... 2024-04-02T15:14:09.975 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:14:09 smithi169 bash[48358]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana.a" found: no such container 2024-04-02T15:14:09.976 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:14:09 smithi169 podman[48291]: 2024-04-02 15:14:09.682859234 +0000 UTC m=+0.617285804 container remove dbb30e649ccfc88ac848190e9f5fadeb9261894ee68a7139a310a309cbd5cedb (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T15:14:09.976 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:14:09 smithi169 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-02T15:14:09.976 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:14:09 smithi169 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-02T15:14:09.976 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:14:09 smithi169 systemd[1]: Stopped Ceph node-exporter.b for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T15:14:09.976 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Apr 02 15:14:09 smithi169 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@node-exporter.b.service: Consumed 5.135s CPU time. 2024-04-02T15:14:10.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:14:10 smithi169 ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a[41919]: t=2024-04-02T15:14:10+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-04-02T15:14:10.264 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:14:10 smithi169 podman[48371]: 2024-04-02 15:14:10.022883351 +0000 UTC m=+0.126289534 container died acd65a3d3b6af3a42f8f7a492670762afce133d0562c32a0270c7d9474e1a902 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a, com.redhat.component=ubi8-container, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, version=8.5, io.openshift.tags=base rhel8, architecture=x86_64, maintainer=Paul Cuzner , distribution-scope=public, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, io.buildah.version=1.24.2) 2024-04-02T15:14:10.522 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:14:10 smithi169 podman[48371]: 2024-04-02 15:14:10.335627513 +0000 UTC m=+0.439033699 container cleanup acd65a3d3b6af3a42f8f7a492670762afce133d0562c32a0270c7d9474e1a902 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, build-date=2022-03-28T10:36:18.413762, io.buildah.version=1.24.2, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, release=236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., architecture=x86_64, maintainer=Paul Cuzner , name=ubi8, description=Ceph Grafana Container, distribution-scope=public) 2024-04-02T15:14:10.523 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:14:10 smithi169 bash[48371]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a 2024-04-02T15:14:10.523 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:14:10 smithi169 bash[48397]: Error: no container with name or ID "ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana.a" found: no such container 2024-04-02T15:14:10.835 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:14:10 smithi169 podman[48385]: 2024-04-02 15:14:10.536805657 +0000 UTC m=+0.514095137 container remove acd65a3d3b6af3a42f8f7a492670762afce133d0562c32a0270c7d9474e1a902 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, name=ubi8, io.buildah.version=1.24.2, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, version=8.5, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., maintainer=Paul Cuzner , io.openshift.tags=base rhel8, build-date=2022-03-28T10:36:18.413762, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, release=236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public) 2024-04-02T15:14:10.835 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:14:10 smithi169 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@grafana.a.service: Deactivated successfully. 2024-04-02T15:14:10.835 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:14:10 smithi169 systemd[1]: Stopped Ceph grafana.a for fd27f2e2-f100-11ee-b647-cb9ed24678a4. 2024-04-02T15:14:10.835 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Apr 02 15:14:10 smithi169 systemd[1]: ceph-fd27f2e2-f100-11ee-b647-cb9ed24678a4@grafana.a.service: Consumed 4.377s CPU time. 2024-04-02T15:14:21.992 DEBUG:teuthology.orchestra.run.smithi042:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T15:14:22.027 DEBUG:teuthology.orchestra.run.smithi169:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T15:14:22.057 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-02T15:14:22.059 DEBUG:teuthology.misc:Transferring archived files from smithi042:/var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/crash to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636039/remote/smithi042/crash 2024-04-02T15:14:22.060 DEBUG:teuthology.orchestra.run.smithi042:> sudo tar c -f - -C /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/crash -- . 2024-04-02T15:14:22.111 INFO:teuthology.orchestra.run.smithi042.stderr:tar: /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-02T15:14:22.112 INFO:teuthology.orchestra.run.smithi042.stderr:tar: Error is not recoverable: exiting now 2024-04-02T15:14:22.113 DEBUG:teuthology.misc:Transferring archived files from smithi169:/var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/crash to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636039/remote/smithi169/crash 2024-04-02T15:14:22.114 DEBUG:teuthology.orchestra.run.smithi169:> sudo tar c -f - -C /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/crash -- . 2024-04-02T15:14:22.149 INFO:teuthology.orchestra.run.smithi169.stderr:tar: /var/lib/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-02T15:14:22.150 INFO:teuthology.orchestra.run.smithi169.stderr:tar: Error is not recoverable: exiting now 2024-04-02T15:14:22.151 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-02T15:14:22.151 DEBUG:teuthology.orchestra.run.smithi042:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-02T15:14:22.184 INFO:teuthology.orchestra.run.smithi042.stdout:2024-04-02T14:57:16.478605+0000 mon.a (mon.0) 165 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-02T15:14:22.184 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-02T15:14:22.184 INFO:tasks.cephadm:Compressing logs... 2024-04-02T15:14:22.185 DEBUG:teuthology.orchestra.run.smithi042:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-02T15:14:22.227 DEBUG:teuthology.orchestra.run.smithi169:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-02T15:14:22.252 INFO:teuthology.orchestra.run.smithi042.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-02T15:14:22.253 INFO:teuthology.orchestra.run.smithi042.stderr:: No such file or directory 2024-04-02T15:14:22.253 INFO:teuthology.orchestra.run.smithi042.stderr:gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-mon.a.log 2024-04-02T15:14:22.253 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.audit.log 2024-04-02T15:14:22.253 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.1.log 2024-04-02T15:14:22.253 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi042.plghvb.log 2024-04-02T15:14:22.254 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.3.log 2024-04-02T15:14:22.266 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi042.plghvb.log: gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-mgr.y.log 2024-04-02T15:14:22.266 INFO:teuthology.orchestra.run.smithi042.stderr: 62.3% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi042.plghvb.log.gz 2024-04-02T15:14:22.266 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/tcmu-runner.log 2024-04-02T15:14:22.266 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.0.log 2024-04-02T15:14:22.266 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/tcmu-runner.log: gzip -5 --verbose 66.2% -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.cephadm.log 2024-04-02T15:14:22.266 INFO:teuthology.orchestra.run.smithi042.stderr: -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/tcmu-runner.log.gz 2024-04-02T15:14:22.267 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.0.log: 93.9% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.audit.log.gz 2024-04-02T15:14:22.267 INFO:teuthology.orchestra.run.smithi042.stderr:gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.log 2024-04-02T15:14:22.267 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.2.log 2024-04-02T15:14:22.267 INFO:teuthology.orchestra.run.smithi042.stderr: 88.5% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-02T15:14:22.267 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.log: 93.5% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.log.gz 2024-04-02T15:14:22.267 INFO:teuthology.orchestra.run.smithi042.stderr:gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-volume.log 2024-04-02T15:14:22.267 INFO:teuthology.orchestra.run.smithi042.stderr:gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-mon.c.log 2024-04-02T15:14:22.267 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.2.log: /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-volume.log: 91.5% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-02T15:14:22.270 INFO:teuthology.orchestra.run.smithi169.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-02T15:14:22.270 INFO:teuthology.orchestra.run.smithi169.stderr:: No such file or directory 2024-04-02T15:14:22.271 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.6.log 2024-04-02T15:14:22.271 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.audit.log 2024-04-02T15:14:22.271 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cephadm.log: gzip -5/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.6.log: --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.4.log 2024-04-02T15:14:22.271 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.7.log 2024-04-02T15:14:22.271 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.5.log 2024-04-02T15:14:22.271 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.4.log: /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-mgr.x.log 2024-04-02T15:14:22.271 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.cephadm.log 2024-04-02T15:14:22.272 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-mgr.x.log: /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.log 2024-04-02T15:14:22.272 INFO:teuthology.orchestra.run.smithi169.stderr: 90.3% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.audit.log.gz 2024-04-02T15:14:22.272 INFO:teuthology.orchestra.run.smithi169.stderr: 80.2% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-02T15:14:22.272 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-volume.log 2024-04-02T15:14:22.272 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.log: gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-mon.b.log 2024-04-02T15:14:22.272 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-volume.log: 88.5% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph.log.gz 2024-04-02T15:14:22.272 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi169.pvuzch.log 2024-04-02T15:14:22.273 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-mon.b.log: 90.5% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-02T15:14:22.273 INFO:teuthology.orchestra.run.smithi169.stderr: 90.9% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-volume.log.gz 2024-04-02T15:14:22.273 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi169.pvuzch.log: 62.4% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi169.pvuzch.log.gz 2024-04-02T15:14:22.273 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-mon.c.log: 91.2% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-volume.log.gz 2024-04-02T15:14:22.282 INFO:teuthology.orchestra.run.smithi169.stderr: 92.5% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-mgr.x.log.gz 2024-04-02T15:14:22.386 INFO:teuthology.orchestra.run.smithi042.stderr: 89.3% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-mgr.y.log.gz 2024-04-02T15:14:22.459 INFO:teuthology.orchestra.run.smithi169.stderr: 92.4% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-mon.b.log.gz 2024-04-02T15:14:22.622 INFO:teuthology.orchestra.run.smithi042.stderr: 92.2% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-mon.c.log.gz 2024-04-02T15:14:22.736 INFO:teuthology.orchestra.run.smithi042.stderr: 93.1% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.2.log.gz 2024-04-02T15:14:22.765 INFO:teuthology.orchestra.run.smithi169.stderr: 93.0% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.6.log.gz 2024-04-02T15:14:22.786 INFO:teuthology.orchestra.run.smithi169.stderr: 93.7% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.7.log.gz 2024-04-02T15:14:22.822 INFO:teuthology.orchestra.run.smithi042.stderr: 93.3% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.0.log.gz 2024-04-02T15:14:22.835 INFO:teuthology.orchestra.run.smithi169.stderr: 93.5% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.4.log.gz 2024-04-02T15:14:22.840 INFO:teuthology.orchestra.run.smithi042.stderr: 91.0% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-mon.a.log.gz 2024-04-02T15:14:22.867 INFO:teuthology.orchestra.run.smithi169.stderr: 93.3% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.5.log.gz 2024-04-02T15:14:22.869 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-04-02T15:14:22.869 INFO:teuthology.orchestra.run.smithi169.stderr:real 0m0.628s 2024-04-02T15:14:22.869 INFO:teuthology.orchestra.run.smithi169.stderr:user 0m2.387s 2024-04-02T15:14:22.869 INFO:teuthology.orchestra.run.smithi169.stderr:sys 0m0.121s 2024-04-02T15:14:23.021 INFO:teuthology.orchestra.run.smithi042.stderr: 93.3% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.1.log.gz 2024-04-02T15:14:23.067 INFO:teuthology.orchestra.run.smithi042.stderr: 93.3% -- replaced with /var/log/ceph/fd27f2e2-f100-11ee-b647-cb9ed24678a4/ceph-osd.3.log.gz 2024-04-02T15:14:23.069 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-04-02T15:14:23.069 INFO:teuthology.orchestra.run.smithi042.stderr:real 0m0.830s 2024-04-02T15:14:23.069 INFO:teuthology.orchestra.run.smithi042.stderr:user 0m3.618s 2024-04-02T15:14:23.069 INFO:teuthology.orchestra.run.smithi042.stderr:sys 0m0.139s 2024-04-02T15:14:23.069 INFO:tasks.cephadm:Archiving logs... 2024-04-02T15:14:23.070 DEBUG:teuthology.misc:Transferring archived files from smithi042:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636039/remote/smithi042/log 2024-04-02T15:14:23.070 DEBUG:teuthology.orchestra.run.smithi042:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-02T15:14:23.671 DEBUG:teuthology.misc:Transferring archived files from smithi169:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636039/remote/smithi169/log 2024-04-02T15:14:23.671 DEBUG:teuthology.orchestra.run.smithi169:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-02T15:14:24.108 INFO:tasks.cephadm:Removing cluster... 2024-04-02T15:14:24.108 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 --force 2024-04-02T15:14:24.392 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 --force 2024-04-02T15:14:24.680 INFO:tasks.cephadm:Removing cephadm ... 2024-04-02T15:14:24.680 DEBUG:teuthology.orchestra.run.smithi042:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-02T15:14:24.696 DEBUG:teuthology.orchestra.run.smithi169:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-02T15:14:24.712 INFO:tasks.cephadm:Teardown complete 2024-04-02T15:14:24.712 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-02T15:14:24.724 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-02T15:14:24.725 DEBUG:teuthology.orchestra.run.smithi042:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T15:14:24.739 DEBUG:teuthology.orchestra.run.smithi169:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T15:14:24.750 INFO:teuthology.orchestra.run.smithi042.stderr:bash: line 1: ntpq: command not found 2024-04-02T15:14:24.755 INFO:teuthology.orchestra.run.smithi042.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T15:14:24.755 INFO:teuthology.orchestra.run.smithi042.stdout:=============================================================================== 2024-04-02T15:14:24.755 INFO:teuthology.orchestra.run.smithi042.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 56 -437us[ -438us] +/- 80ms 2024-04-02T15:14:24.755 INFO:teuthology.orchestra.run.smithi042.stdout:^* hv02.front.sepia.ceph.com 3 6 377 52 -828us[ -829us] +/- 68ms 2024-04-02T15:14:24.755 INFO:teuthology.orchestra.run.smithi042.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 54 +1550us[+1549us] +/- 85ms 2024-04-02T15:14:24.755 INFO:teuthology.orchestra.run.smithi042.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T15:14:24.765 INFO:teuthology.orchestra.run.smithi169.stderr:bash: line 1: ntpq: command not found 2024-04-02T15:14:24.770 INFO:teuthology.orchestra.run.smithi169.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T15:14:24.770 INFO:teuthology.orchestra.run.smithi169.stdout:=============================================================================== 2024-04-02T15:14:24.770 INFO:teuthology.orchestra.run.smithi169.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 55 -422us[ -422us] +/- 80ms 2024-04-02T15:14:24.770 INFO:teuthology.orchestra.run.smithi169.stdout:^* hv02.front.sepia.ceph.com 3 6 377 58 -837us[ -823us] +/- 68ms 2024-04-02T15:14:24.771 INFO:teuthology.orchestra.run.smithi169.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 54 +1557us[+1557us] +/- 85ms 2024-04-02T15:14:24.771 INFO:teuthology.orchestra.run.smithi169.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T15:14:24.771 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-02T15:14:24.781 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-02T15:14:24.782 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-02T15:14:24.791 DEBUG:teuthology.orchestra.run.smithi042:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-02T15:14:24.872 DEBUG:teuthology.orchestra.run.smithi169:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-02T15:14:24.943 DEBUG:teuthology.orchestra.run.smithi042:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-02T15:14:24.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T15:14:24.973 DEBUG:teuthology.orchestra.run.smithi169:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-02T15:14:25.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T15:14:25.004 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-02T15:14:25.017 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-02T15:14:25.032 INFO:teuthology.task.internal:Duration was 1423.323025 seconds 2024-04-02T15:14:25.033 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-02T15:14:25.044 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-02T15:14:25.044 DEBUG:teuthology.orchestra.run.smithi042:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-02T15:14:25.047 DEBUG:teuthology.orchestra.run.smithi169:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-02T15:14:25.091 INFO:teuthology.orchestra.run.smithi169.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T15:14:25.092 INFO:teuthology.orchestra.run.smithi042.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T15:14:25.520 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-02T15:14:25.520 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi042.front.sepia.ceph.com 2024-04-02T15:14:25.521 DEBUG:teuthology.orchestra.run.smithi042:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-02T15:14:25.583 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi169.front.sepia.ceph.com 2024-04-02T15:14:25.583 DEBUG:teuthology.orchestra.run.smithi169:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-02T15:14:25.605 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-02T15:14:25.606 DEBUG:teuthology.orchestra.run.smithi042:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-02T15:14:25.626 DEBUG:teuthology.orchestra.run.smithi169:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-02T15:14:25.676 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-02T15:14:25.677 DEBUG:teuthology.orchestra.run.smithi042:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-02T15:14:25.698 DEBUG:teuthology.orchestra.run.smithi169:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-02T15:14:26.008 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-02T15:14:26.023 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-02T15:14:26.023 DEBUG:teuthology.orchestra.run.smithi042:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-02T15:14:26.051 DEBUG:teuthology.orchestra.run.smithi169:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-02T15:14:26.083 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-02T15:14:26.095 DEBUG:teuthology.orchestra.run.smithi042:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-02T15:14:26.097 DEBUG:teuthology.orchestra.run.smithi169:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-02T15:14:26.120 INFO:teuthology.orchestra.run.smithi042.stdout:kernel.core_pattern = core 2024-04-02T15:14:26.150 INFO:teuthology.orchestra.run.smithi169.stdout:kernel.core_pattern = core 2024-04-02T15:14:26.181 DEBUG:teuthology.orchestra.run.smithi042:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-02T15:14:26.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T15:14:26.199 DEBUG:teuthology.orchestra.run.smithi169:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-02T15:14:26.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T15:14:26.236 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-02T15:14:26.247 INFO:teuthology.task.internal:Transferring archived files... 2024-04-02T15:14:26.248 DEBUG:teuthology.misc:Transferring archived files from smithi042:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636039/remote/smithi042 2024-04-02T15:14:26.248 DEBUG:teuthology.orchestra.run.smithi042:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-02T15:14:26.298 DEBUG:teuthology.misc:Transferring archived files from smithi169:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636039/remote/smithi169 2024-04-02T15:14:26.298 DEBUG:teuthology.orchestra.run.smithi169:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-02T15:14:26.338 INFO:teuthology.task.internal:Removing archive directory... 2024-04-02T15:14:26.339 DEBUG:teuthology.orchestra.run.smithi042:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-02T15:14:26.341 DEBUG:teuthology.orchestra.run.smithi169:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-02T15:14:26.391 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-02T15:14:26.404 INFO:teuthology.task.internal:Not uploading archives. 2024-04-02T15:14:26.404 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-02T15:14:26.416 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-02T15:14:26.416 DEBUG:teuthology.orchestra.run.smithi042:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-02T15:14:26.419 DEBUG:teuthology.orchestra.run.smithi169:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-02T15:14:26.431 INFO:teuthology.orchestra.run.smithi042.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 2 15:14 /home/ubuntu/cephtest 2024-04-02T15:14:26.446 INFO:teuthology.orchestra.run.smithi169.stdout: 265305 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 2 15:14 /home/ubuntu/cephtest 2024-04-02T15:14:26.448 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-02T15:14:26.466 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-02T15:14:26.493 INFO:teuthology.nuke:Checking targets against current locks 2024-04-02T15:14:26.522 DEBUG:teuthology.nuke:shortname: smithi042 2024-04-02T15:14:26.523 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-02T15:14:26.549 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi042.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636039', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 14:42:05.927714', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKpnlmMUlvy2mfaktOdc7N9aQbBz7qTP7ch5a3MsEvt3PxsGNa5DgrrQtZ7KKeWsXqT0iw243939Ae8VWa2J8CA='} 2024-04-02T15:14:26.554 DEBUG:teuthology.nuke:shortname: smithi169 2024-04-02T15:14:26.554 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-02T15:14:26.578 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi169.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636039', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 14:42:05.929639', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEBuLnltjtfvWtb3uWYalmbxm0omnfwG45avdad+XyG42yEv9QBPNeW2LNIp8DKlYhldm4psBIr49Y0sdKzBoNk='} 2024-04-02T15:14:26.617 INFO:teuthology.orchestra.console.smithi042:Power off 2024-04-02T15:14:26.617 DEBUG:teuthology.orchestra.console.smithi042:pexpect command: ipmitool -H smithi042.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-02T15:14:26.645 DEBUG:teuthology.orchestra.console.smithi042:power off output: Chassis Power Control: Down/Off 2024-04-02T15:14:26.645 DEBUG:teuthology.orchestra.console.smithi042:pexpect command: ipmitool -H smithi042.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T15:14:26.666 INFO:teuthology.orchestra.console.smithi169:Power off 2024-04-02T15:14:26.666 DEBUG:teuthology.orchestra.console.smithi169:pexpect command: ipmitool -H smithi169.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-02T15:14:26.676 DEBUG:teuthology.orchestra.console.smithi042:check power output: Chassis Power is on 2024-04-02T15:14:26.692 DEBUG:teuthology.orchestra.console.smithi169:power off output: Chassis Power Control: Down/Off 2024-04-02T15:14:26.692 DEBUG:teuthology.orchestra.console.smithi169:pexpect command: ipmitool -H smithi169.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T15:14:26.730 DEBUG:teuthology.orchestra.console.smithi169:check power output: Chassis Power is on 2024-04-02T15:14:30.677 DEBUG:teuthology.orchestra.console.smithi042:pexpect command: ipmitool -H smithi042.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T15:14:30.731 DEBUG:teuthology.orchestra.console.smithi169:pexpect command: ipmitool -H smithi169.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T15:14:30.789 DEBUG:teuthology.orchestra.console.smithi042:check power output: Chassis Power is on 2024-04-02T15:14:30.845 DEBUG:teuthology.orchestra.console.smithi169:check power output: Chassis Power is on 2024-04-02T15:14:34.794 DEBUG:teuthology.orchestra.console.smithi042:pexpect command: ipmitool -H smithi042.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T15:14:34.847 DEBUG:teuthology.orchestra.console.smithi169:pexpect command: ipmitool -H smithi169.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T15:14:34.918 DEBUG:teuthology.orchestra.console.smithi042:check power output: Chassis Power is off 2024-04-02T15:14:34.960 DEBUG:teuthology.orchestra.console.smithi169:check power output: Chassis Power is on 2024-04-02T15:14:35.020 INFO:teuthology.orchestra.console.smithi042:Power off completed 2024-04-02T15:14:38.965 DEBUG:teuthology.orchestra.console.smithi169:pexpect command: ipmitool -H smithi169.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T15:14:39.079 DEBUG:teuthology.orchestra.console.smithi169:check power output: Chassis Power is off 2024-04-02T15:14:39.179 INFO:teuthology.orchestra.console.smithi169:Power off completed 2024-04-02T15:14:39.284 INFO:teuthology.run:Summary data: description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} duration: 1423.3230247497559 failure_reason: 'Command failed on smithi042 with status 22: ''sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fd27f2e2-f100-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c \''ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1\''''' owner: scheduled_gabrioux@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=f22d0fd28a53454f9bd4c885fbe87321 status: fail success: false 2024-04-02T15:14:39.285 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-02T15:14:39.375 INFO:teuthology.run:FAIL